Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 5681583

Browse files
KrinkleKrinkle
committedJan 23, 2025·
WIP
1 parent da1e629 commit 5681583

14 files changed

+643
-157
lines changed
 

‎.github/workflows/CI.yaml

+4-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,10 @@ jobs:
5656
- run: npm test
5757

5858
- name: Check system browsers
59-
run: node bin/qtap.js -v -b firefox -b chrome -b chromium -b edge test/pass.html
59+
# Increase -connect-timeout from 60s because on GitHub CI's free tier,
60+
# there is low CPU concurrency, which means the last browser basically
61+
# has to wait for earlier ones to finish.
62+
run: node bin/qtap.js -v -b firefox -b chrome -b chromium -b edge --connect-timeout 100 test/pass.html
6063

6164
- name: Check system browsers (Safari)
6265
if: ${{ runner.os == 'macOS' }}

‎bin/qtap.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ program
4949
},
5050
60
5151
)
52+
.option('-r, --reporter <reporter>')
5253
.option('-w, --watch', 'Watch files for changes and re-run the test suite.')
5354
.option('-v, --verbose', 'Enable verbose debug logging.')
5455
.option('-V, --version', 'Display version number.')
@@ -74,13 +75,14 @@ if (opts.version) {
7475
});
7576

7677
try {
77-
const exitCode = await qtap.run(opts.browser, program.args, {
78+
const result = await qtap.runWaitFor(opts.browser, program.args, {
7879
config: opts.config,
7980
timeout: opts.timeout,
8081
connectTimeout: opts.connectTimeout,
82+
reporter: opts.reporter,
8183
verbose: opts.verbose
8284
});
83-
process.exit(exitCode);
85+
process.exit(result.exitCode);
8486
} catch (e) {
8587
console.error(e);
8688
process.exit(1);

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
"eslint": "~8.57.1",
3535
"eslint-config-semistandard": "~17.0.0",
3636
"eslint-plugin-qunit": "^8.1.2",
37-
"qunit": "2.23.1",
37+
"qunit": "2.24.0",
3838
"semistandard": "~17.0.0",
3939
"typescript": "5.7.3"
4040
},

‎spinners.json

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{
2+
"dots": {
3+
"interval": 80,
4+
"frames": [
5+
"⠋",
6+
"⠙",
7+
"⠹",
8+
"⠸",
9+
"⠼",
10+
"⠴",
11+
"⠦",
12+
"⠧",
13+
"⠇",
14+
"⠏"
15+
]
16+
},
17+
"dotsCircle": {
18+
"interval": 80,
19+
"frames": [
20+
"⢎ ",
21+
"⠎⠁",
22+
"⠊⠑",
23+
"⠈⠱",
24+
" ⡱",
25+
"⢀⡰",
26+
"⢄⡠",
27+
"⢆⡀"
28+
]
29+
},
30+
"toggle10": {
31+
"interval": 100,
32+
"frames": [
33+
"㊂",
34+
"㊀",
35+
"㊁"
36+
]
37+
},
38+
"pong": {
39+
"interval": 80,
40+
"frames": [
41+
"▐⠂ ▌",
42+
"▐⠈ ▌",
43+
"▐ ⠂ ▌",
44+
"▐ ⠠ ▌",
45+
"▐ ⡀ ▌",
46+
"▐ ⠠ ▌",
47+
"▐ ⠂ ▌",
48+
"▐ ⠈ ▌",
49+
"▐ ⠂ ▌",
50+
"▐ ⠠ ▌",
51+
"▐ ⡀ ▌",
52+
"▐ ⠠ ▌",
53+
"▐ ⠂ ▌",
54+
"▐ ⠈ ▌",
55+
"▐ ⠂▌",
56+
"▐ ⠠▌",
57+
"▐ ⡀▌",
58+
"▐ ⠠ ▌",
59+
"▐ ⠂ ▌",
60+
"▐ ⠈ ▌",
61+
"▐ ⠂ ▌",
62+
"▐ ⠠ ▌",
63+
"▐ ⡀ ▌",
64+
"▐ ⠠ ▌",
65+
"▐ ⠂ ▌",
66+
"▐ ⠈ ▌",
67+
"▐ ⠂ ▌",
68+
"▐ ⠠ ▌",
69+
"▐ ⡀ ▌",
70+
"▐⠠ ▌"
71+
]
72+
},
73+
"layer": {
74+
"interval": 150,
75+
"frames": [
76+
"-",
77+
"=",
78+
"≡"
79+
]
80+
}
81+
}

‎src/qtap.js

+105-45
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
'use strict';
22

3+
import { EventEmitter } from 'node:events';
34
import util from 'node:util';
45
import path from 'node:path';
56

67
import kleur from 'kleur';
78
import browsers from './browsers.js';
9+
import reporters from './reporters.js';
810
import { ControlServer } from './server.js';
911

1012
/**
@@ -83,6 +85,7 @@ function makeLogger (defaultChannel, printDebug, verbose = false) {
8385
* @property {number} [timeout=30] How long a browser may be quiet between results.
8486
* @property {number} [connectTimeout=60] How many seconds a browser may take to start up.
8587
* @property {boolean} [verbose=false]
88+
* @property {string} [reporter]
8689
* @property {string} [cwd=process.cwd()] Base directory to interpret test file paths
8790
* relative to. Ignored if testing from URLs.
8891
* @property {Function} [printDebug=console.error]
@@ -93,9 +96,9 @@ function makeLogger (defaultChannel, printDebug, verbose = false) {
9396
* to a built-in browser from QTap, or to a key in the optional `config.browsers` object.
9497
* @param {string|string[]} files Files and/or URLs.
9598
* @param {qtap.RunOptions} [options]
96-
* @return {Promise<number>} Exit code. 0 is success, 1 is failed.
99+
* @return {EventEmitter}
97100
*/
98-
async function run (browserNames, files, options = {}) {
101+
function run (browserNames, files, options = {}) {
99102
if (typeof browserNames === 'string') browserNames = [browserNames];
100103
if (typeof files === 'string') files = [files];
101104

@@ -104,67 +107,124 @@ async function run (browserNames, files, options = {}) {
104107
options.printDebug || console.error,
105108
options.verbose
106109
);
110+
const eventbus = new EventEmitter();
111+
112+
if (options.reporter) {
113+
if (options.reporter in reporters) {
114+
logger.debug('reporter_init', options.reporter);
115+
reporters[options.reporter](eventbus);
116+
} else {
117+
logger.warning('reporter_unknown', options.reporter);
118+
}
119+
}
107120

108121
const servers = [];
109122
for (const file of files) {
110-
servers.push(new ControlServer(options.cwd, file, logger, {
123+
servers.push(new ControlServer(options.cwd, file, eventbus, logger, {
111124
idleTimeout: options.timeout,
112125
connectTimeout: options.connectTimeout
113126
}));
114127
}
115128

116-
// TODO: Add test for config file not found
117-
// TODO: Add test for config file with runtime errors
118-
// TODO: Add test for relative config file without leading `./`, handled by process.resolve()
119-
let config;
120-
if (typeof options.config === 'string') {
121-
logger.debug('load_config', options.config);
122-
config = (await import(path.resolve(process.cwd(), options.config))).default;
123-
}
124-
const globalController = new AbortController();
125-
const globalSignal = globalController.signal;
126-
127-
const browerPromises = [];
128-
for (const browserName of browserNames) {
129-
logger.debug('get_browser', browserName);
130-
const browserFn = browsers[browserName] || config?.browsers?.[browserName];
131-
if (typeof browserFn !== 'function') {
132-
throw new Error('Unknown browser ' + browserName);
129+
const runPromise = (async () => {
130+
// TODO: Add test for config file not found
131+
// TODO: Add test for config file with runtime errors
132+
// TODO: Add test for relative config file without leading `./`, handled by process.resolve()
133+
let config;
134+
if (typeof options.config === 'string') {
135+
logger.debug('load_config', options.config);
136+
config = (await import(path.resolve(process.cwd(), options.config))).default;
133137
}
134-
for (const server of servers) {
135-
// Each launchBrowser() returns a Promise that settles when the browser exits.
136-
// Launch concurrently, and await afterwards.
137-
browerPromises.push(server.launchBrowser(browserFn, browserName, globalSignal));
138+
const globalController = new AbortController();
139+
const globalSignal = globalController.signal;
140+
141+
const browerPromises = [];
142+
for (const browserName of browserNames) {
143+
logger.debug('get_browser', browserName);
144+
const browserFn = browsers[browserName] || config?.browsers?.[browserName];
145+
if (typeof browserFn !== 'function') {
146+
throw new Error('Unknown browser ' + browserName);
147+
}
148+
for (const server of servers) {
149+
// Each launchBrowser() returns a Promise that settles when the browser exits.
150+
// Launch concurrently, and await afterwards.
151+
browerPromises.push(server.launchBrowser(browserFn, browserName, globalSignal));
152+
}
138153
}
139-
}
140-
141-
try {
142-
// Wait for all tests and browsers to finish/stop, regardless of errors thrown,
143-
// to avoid dangling browser processes.
144-
await Promise.allSettled(browerPromises);
145154

146-
// Re-wait, this time letting the first of any errors bubble up.
147-
for (const browerPromise of browerPromises) {
148-
await browerPromise;
155+
const result = {
156+
ok: true,
157+
exitCode: 0
158+
};
159+
eventbus.on('clientbail', () => {
160+
result.ok = false;
161+
result.exitCode = 1;
162+
});
163+
eventbus.on('clientresult', (event) => {
164+
if (!event.result.ok) {
165+
result.ok = false;
166+
result.exitCode = 1;
167+
}
168+
});
169+
170+
try {
171+
// Wait for all tests and browsers to finish/stop, regardless of errors thrown,
172+
// to avoid dangling browser processes.
173+
await Promise.allSettled(browerPromises);
174+
175+
// Re-wait, this time letting the first of any errors bubble up.
176+
for (const browerPromise of browerPromises) {
177+
await browerPromise;
178+
}
179+
180+
logger.debug('shared_cleanup', 'Invoke global signal to clean up shared resources');
181+
globalController.abort();
182+
} finally {
183+
// Make sure we close our server even if the above throws, so that Node.js
184+
// may naturally exit (no open ports remaining)
185+
for (const server of servers) {
186+
server.close();
187+
}
149188
}
150189

151-
logger.debug('shared_cleanup', 'Invoke global signal to clean up shared resources');
152-
globalController.abort();
153-
} finally {
154-
// Make sure we close our server even if the above throws, so that Node.js
155-
// may naturally exit (no open ports remaining)
156-
for (const server of servers) {
157-
server.close();
158-
}
159-
}
190+
eventbus.emit('finish', result);
191+
})();
192+
runPromise.catch((error) => {
193+
// Node.js automatically ensures users cannot forget to listen for the 'error' event.
194+
// For this reason, runWaitFor() is a separate method, because that converts the
195+
// 'error' event into a rejected Promise. If we created that Promise as part of run()
196+
// like `return {eventbus, promise}`), then we loose this useful detection, because
197+
// we'd have already listened for it. Plus, it causes an unhandledRejection error
198+
// for those that only want the events and not the Promise.
199+
eventbus.emit('error', error);
200+
});
201+
202+
return eventbus;
203+
}
160204

161-
// TODO: Set exit status to 1 on failures, to ease programmatic use and testing.
162-
// TODO: Return an event emitter for custom reporting via programmatic use.
163-
return 0;
205+
/**
206+
* Same as run() but can awaited.
207+
*
208+
* Use this if all you want is a boolean result and/or if you use the 'reporter'
209+
* option for any output/display. For detailed events, call run() instead.
210+
*
211+
* @return {Promise<{ok: boolean, exitCode: number}>}
212+
* - ok: true for success, false for failure.
213+
* - exitCode: 0 for success, 1 for failure.
214+
*/
215+
async function runWaitFor (browserNames, files, options = {}) {
216+
const eventbus = run(browserNames, files, options);
217+
218+
const result = await new Promise((resolve, reject) => {
219+
eventbus.on('finish', resolve);
220+
eventbus.on('error', reject);
221+
});
222+
return result;
164223
}
165224

166225
export default {
167226
run,
227+
runWaitFor,
168228

169229
browsers,
170230
LocalBrowser: browsers.LocalBrowser,

‎src/reporters.js

+195
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
function dynamic (eventbus) {
2+
/**
3+
* @typedef {Object} ClientState
4+
* @property {string} clientId
5+
* @property {string} displayName
6+
* @property {string} status
7+
* @property {null|string} lastline
8+
* @property {Array} failures
9+
*/
10+
11+
/** @type {Object<string,ClientState[]>} */
12+
const clientsByFile = Object.create(null);
13+
/** @type {Object<string,ClientState>} */
14+
const clientsById = Object.create(null);
15+
16+
let screen = '';
17+
18+
function render () {
19+
const icons = { waiting: '⢎', progress: '⠧', success: '✔', failure: '✘' };
20+
let str = '';
21+
for (const testFile in clientsByFile) {
22+
str += `\nRunning ${testFile}\n`;
23+
for (const client of clientsByFile[testFile]) {
24+
str += `* ${client.displayName} ${icons[client.status]} ${client.lastline || ''}\n`;
25+
}
26+
}
27+
28+
if (screen) {
29+
const oldHeight = screen.split('\n').length;
30+
for (let i = 1; i < oldHeight; i++) {
31+
process.stdout.write('\x1b[A\x1b[K');
32+
}
33+
}
34+
35+
process.stdout.write(str);
36+
screen = str;
37+
}
38+
39+
eventbus.on('clientcreate', (event) => {
40+
const client = {
41+
clientId: event.clientId,
42+
displayName: event.displayName,
43+
status: 'waiting',
44+
lastline: null,
45+
failures: []
46+
};
47+
48+
clientsByFile[event.testFile] ??= [];
49+
clientsByFile[event.testFile].push(client);
50+
clientsById[event.clientId] = client;
51+
render();
52+
});
53+
54+
eventbus.on('clientonline', (event) => {
55+
clientsById[event.clientId].status = 'progress';
56+
render();
57+
});
58+
59+
eventbus.on('clientsample', (event) => {
60+
clientsById[event.clientId].lastline = event.line;
61+
render();
62+
});
63+
64+
// eventbus.on('clientend', (event) => {
65+
// clientsById[event.clientId].status = 'failure';
66+
// clientsById[event.clientId].lastline = event.reason;
67+
// render();
68+
// });
69+
}
70+
71+
function plain (eventbus) {
72+
// Testing <file> in <browser>
73+
// Testing N file(s) in N browser(s)
74+
75+
eventbus.on('clientcreate', (event) => {
76+
console.log(`Testing ${event.testFile} in ${event.displayName}`);
77+
});
78+
eventbus.on('clientonline', () => {
79+
console.log('Connected!');
80+
});
81+
eventbus.on('clientsample', () => {
82+
console.log('.');
83+
});
84+
eventbus.on('clientbail', () => {
85+
console.log('Bail out!');
86+
});
87+
eventbus.on('clientresult', (event) => {
88+
console.log('Result', event);
89+
});
90+
}
91+
92+
const isTTY = false; // process.stdout.isTTY && process.env.TERM !== 'dumb';
93+
const minimal = isTTY ? dynamic : plain;
94+
95+
function noop (_eventbus) {}
96+
97+
export default { noop, minimal };
98+
99+
// TODO: Default: TAP where each browser is 1 virtual test in case of success.
100+
// TODO: Verbose: TAP forwarded, test names prepended with [browsername].
101+
// TODO: Failures are shown either way, with prepended names.
102+
// TODO: On "runEnd", report wall-clock runtime
103+
// Default: No-op, as overall TAP line as single test (above) can contain runtime
104+
// Verbose: Output comment indicatinh browser done, and test runtime.
105+
106+
/*
107+
108+
Running /test/pass.html
109+
* Firefox ⠧ ok 3 Baz > another thing
110+
* Chrome ⠧ ok 1 Foo bar
111+
112+
===============================================================
113+
114+
Running /test/pass.html
115+
* Firefox ✔ Ran 4 tests in 42ms
116+
* Chrome ✔ Completed 4 tests in 42ms
117+
118+
===============================================================
119+
120+
Running /test/fail.html
121+
* Firefox ⠧ not ok 2 example > hello fail
122+
* Chrome ⠧ ok 1 Foo bar
123+
124+
There was 1 failure
125+
126+
1. Firefox - example > hello fail
127+
---
128+
actual : foo
129+
expected: bar
130+
...
131+
132+
===============================================================
133+
134+
Running /test/fail.html
135+
* Firefox ⠧ ok 3 Quux # update to next result, but with red spinner
136+
* Chrome ⠧ ok 3 Quux
137+
138+
There were 2 failures
139+
140+
1. Firefox - example > hello fail
141+
---
142+
actual : foo
143+
expected: bar
144+
...
145+
146+
2. Chrome - example > hello fail
147+
---
148+
actual : foo
149+
expected: bar
150+
...
151+
152+
===============================================================
153+
154+
Running /test/fail.html
155+
* Firefox ✘ Ran 3 tests in 42ms (1 failed)
156+
* Chrome ✘ Completed 3 tests in 42ms (1 failed)
157+
* Chrome ✘ 1 failed test
158+
159+
There were 2 failures
160+
161+
1. Firefox - example > hello fail
162+
---
163+
actual : foo
164+
expected: bar
165+
...
166+
167+
2. Chrome - example > hello fail
168+
---
169+
actual : foo
170+
expected: bar
171+
...
172+
173+
===============================================================
174+
175+
Running /test/timeout.html
176+
* Firefox ⠧ ok 2 Baz > this thing
177+
* Chrome ⠧ ok 1 Foo bar
178+
179+
===============================================================
180+
181+
Running /test/timeout.html
182+
* Firefox ? Test timed out after 30s of inactivity
183+
* Chrome ? Test timed out after 30s of inactivity
184+
185+
===============================================================
186+
187+
Running /test/connect-timeout.html
188+
* Firefox ⢎ Waiting...
189+
* Chrome ⢎ Waiting...
190+
191+
Running /test/connect-timeout.html
192+
* Firefox ? Browser did not start within 60s
193+
* Chrome ? Browser did not start within 60s
194+
195+
*/

‎src/server.js

+76-95
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import fs from 'node:fs';
44
import fsPromises from 'node:fs/promises';
55
import http from 'node:http';
66
import path from 'node:path';
7+
import { pathToFileURL } from 'node:url';
78

89
import { fnToStr, qtapClientHead, qtapClientBody } from './client.js';
910
import * as util from './util.js';
1011
import tapFinished from './tap-finished.js';
12+
/** @import events from 'node:events' */
1113
/** @import { Logger } from './qtap.js' */
1214

1315
const QTAP_DEBUG = process.env.QTAP_DEBUG === '1';
@@ -19,12 +21,13 @@ class ControlServer {
1921
/**
2022
* @param {string|undefined} cwd
2123
* @param {string} testFile File path or URL
24+
* @param {events.EventEmitter} eventbus
2225
* @param {Logger} logger
2326
* @param {Object} options
2427
* @param {number|undefined} options.idleTimeout
2528
* @param {number|undefined} options.connectTimeout
2629
*/
27-
constructor (cwd, testFile, logger, options) {
30+
constructor (cwd, testFile, eventbus, logger, options) {
2831
this.logger = logger.channel('qtap_server_' + ControlServer.nextServerId++);
2932

3033
// For `qtap <url>`, default root to cwd (unused).
@@ -50,11 +53,24 @@ class ControlServer {
5053
if (!testFile || testFile.startsWith('..')) {
5154
throw new Error(`Cannot serve ${testFile} from ${root}`);
5255
}
56+
// Normalize \backslash to POSIX slash, but only on Windows
57+
// * To use as-is in URLs (launchBrowser/qtapUrlPath).
58+
// * Stable values in reporter output text.
59+
// * Stable values in event data.
60+
// * Only on Windows (pathToFileURL chooses automatically),
61+
// because on POSIX, backslash is a valid character to use in
62+
// in a file name, which we must not replace with forward slash.
63+
const rootUrlPathname = pathToFileURL(root).pathname;
64+
const fileUrlPathname = pathToFileURL(testFileAbsolute).pathname;
65+
testFile = fileUrlPathname
66+
.replace(rootUrlPathname, '')
67+
.replace(/^\/+/, '');
5368
this.logger.debug('server_testfile_normalized', testFile);
5469
}
5570

5671
this.root = root;
5772
this.testFile = testFile;
73+
this.eventbus = eventbus;
5874
this.idleTimeout = options.idleTimeout || 30;
5975
this.connectTimeout = options.connectTimeout || 60;
6076

@@ -116,17 +132,18 @@ class ControlServer {
116132
/** @return {Promise<string>} HTML */
117133
async fetchTestFile (file) {
118134
// fetch() does not yet support file URLs (as of Node.js 21).
119-
return this.isURL(file)
120-
? (await (await fetch(file)).text())
121-
: (await fsPromises.readFile(file)).toString();
135+
if (this.isURL(file)) {
136+
this.logger.debug('testfile_fetch', `Requesting a copy of ${file}`);
137+
return await (await fetch(file)).text();
138+
} else {
139+
this.logger.debug('testfile_read', `Reading file contents from ${file}`);
140+
return (await fsPromises.readFile(file)).toString();
141+
}
122142
}
123143

124144
async launchBrowser (browserFn, browserName, globalSignal) {
125145
const clientId = 'client_' + ControlServer.nextClientId++;
126146
const logger = this.logger.channel(`qtap_browser_${clientId}_${browserName}`);
127-
128-
// TODO: Remove `summary` in favour of `eventbus`
129-
const summary = { ok: true };
130147
let clientIdleTimer = null;
131148

132149
const controller = new AbortController();
@@ -139,28 +156,49 @@ class ControlServer {
139156
});
140157
}
141158

142-
// Reasons to stop a browser, whichever comes first:
143-
// 1. tap-finished.
144-
// 2. tap-parser 'bailout' event (client knows it crashed).
145-
// 3. timeout (client didn't start, lost connection, or unknowingly crashed).
146-
const stopBrowser = async (messageCode) => {
159+
/**
160+
* Reasons to stop a browser, whichever comes first:
161+
* 1. tap-finished.
162+
* 2. tap-parser 'bailout' event (client knows it crashed).
163+
* 3. timeout (client didn't start, lost connection, or unknowingly crashed).
164+
*
165+
* @param {string} messageCode
166+
* @param {string} [problem]
167+
* @param {Object} [result]
168+
*/
169+
const stopBrowser = async (messageCode, problem = '', result = null) => {
147170
// Ignore any duplicate or late reasons to stop
148171
if (!this.browsers.has(clientId)) return;
149172

150173
clearTimeout(clientIdleTimer);
151174
this.browsers.delete(clientId);
152175
controller.abort(`QTap: ${messageCode}`);
176+
177+
if (result) {
178+
this.eventbus.emit('clientresult', {
179+
clientId,
180+
result
181+
});
182+
} else {
183+
this.eventbus.emit('clientbail', { clientId, reason: problem });
184+
}
153185
};
154186

155-
const tapParser = tapFinished({ wait: 0 }, () => {
156-
logger.debug('browser_tap_finished', 'Test run finished, stopping browser');
157-
stopBrowser('browser_tap_finished');
187+
const tapParser = tapFinished({ wait: 0 }, (finalResult) => {
188+
const result = {
189+
ok: finalResult.ok,
190+
total: finalResult.count,
191+
passed: finalResult.pass,
192+
failed: finalResult.failures.length
193+
// Tests completed, X passed [, X failed ] [, X skipped ].
194+
};
195+
logger.debug('browser_tap_finished', 'Test run finished, stopping browser', result);
196+
stopBrowser('browser_tap_finished', '', result);
158197
});
159198

160199
tapParser.on('bailout', (reason) => {
161200
logger.warning('browser_tap_bailout', `Test run bailed, stopping browser. Reason: ${reason}`);
162-
summary.ok = false;
163-
stopBrowser('browser_tap_bailout');
201+
stopBrowser('browser_tap_bailout', reason);
164202
});
165203
// Debugging
166204
// tapParser.on('line', logger.debug.bind(logger, 'browser_tap_line'));
@@ -183,21 +221,19 @@ class ControlServer {
183221
// Node.js/V8 natively allocating many timers when processing large batches of test results.
184222
// Instead, merely store performance.now() and check that periodically.
185223
// TODO: Write test for --connect-timeout by using a no-op browser.
186-
const TIMEOUT_CHECK_MS = 1000;
224+
const TIMEOUT_CHECK_MS = 100;
187225
const browserStart = performance.now();
188226
const qtapCheckTimeout = () => {
189227
if (!browser.clientIdleActive) {
190228
if ((performance.now() - browserStart) > (this.connectTimeout * 1000)) {
191229
logger.warning('browser_connect_timeout', `Browser did not start within ${this.connectTimeout}s, stopping browser`);
192-
summary.ok = false;
193-
stopBrowser('browser_connect_timeout');
230+
stopBrowser('browser_connect_timeout', `Browser did not start within ${this.connectTimeout}s`);
194231
return;
195232
}
196233
} else {
197234
if ((performance.now() - browser.clientIdleActive) > (this.idleTimeout * 1000)) {
198235
logger.warning('browser_idle_timeout', `Browser idle for ${this.idleTimeout}s, stopping browser`);
199-
summary.ok = false;
200-
stopBrowser('browser_idle_timeout');
236+
stopBrowser('browser_idle_timeout', `Browser idle for ${this.idleTimeout}s`);
201237
return;
202238
}
203239
}
@@ -228,34 +264,37 @@ class ControlServer {
228264
tmpUrl.searchParams.set('qtap_clientId', clientId);
229265
qtapUrlPath = tmpUrl.pathname + tmpUrl.search;
230266
} else {
231-
qtapUrlPath = '/'
232-
+ (this.testFile
233-
.replace(/^\/+/, '')
234-
// TODO: Add test case to confirm Windows-file to POSIX-URL normalization.
235-
.replace(/\\/g, '/')
236-
)
237-
+ '?qtap_clientId=' + clientId;
267+
qtapUrlPath = '/' + this.testFile + '?qtap_clientId=' + clientId;
238268
}
239269

240270
const url = await this.getProxyBase() + qtapUrlPath;
241271
const signals = { browser: signal, global: globalSignal };
242272

243273
try {
244274
logger.debug('browser_launch_call');
245-
await browserFn(url, signals, logger);
275+
276+
// Separate calling browserFn() from awaiting so that we can emit 'createclient'
277+
// right after calling it (which may set Browser.displayName). If we await here,
278+
// the event would be emitted when the client is done instead of when it starts.
279+
const browerPromise = browserFn(url, signals, logger);
280+
this.eventbus.emit('clientcreate', {
281+
testFile: this.testFile,
282+
clientId,
283+
browserName,
284+
displayName: browser.getDisplayName()
285+
});
286+
await browerPromise;
246287

247288
// This stopBrowser() is most likely a no-op (e.g. if we received test results
248289
// or some error, and we asked the browser to stop). Just in case the browser
249290
// ended by itself, call it again here so that we can convey it as an error
250291
// if it was still running from our POV.
251292
logger.debug('browser_launch_exit');
252-
stopBrowser('browser_launch_exit');
293+
stopBrowser('browser_launch_exit', 'Browser ended unexpectedly');
253294
} catch (e) {
254-
// TODO: Report error to TAP. Eg. "No executable found"
255-
// TODO: logger.warning('browser_launch_exit', err); but catch in qtap.js?
256295
if (!signal.aborted) {
257296
logger.warning('browser_launch_error', e);
258-
stopBrowser('browser_launch_error');
297+
stopBrowser('browser_launch_error', 'Browser ended unexpectedly');
259298
throw e;
260299
}
261300
}
@@ -267,7 +306,6 @@ class ControlServer {
267306

268307
let headInjectHtml = `<script>(${fnToStr(qtapClientHead, qtapUrl)})();</script>`;
269308

270-
// Add <base> tag so that /test/index.html can refer to files relative to it,
271309
// and URL-based files can fetch their resources directly from the original server.
272310
// * Prepend as early as possible. If the file has its own <base>, theirs will
273311
// come later and correctly "win" by applying last (after ours).
@@ -277,6 +315,7 @@ class ControlServer {
277315
}
278316

279317
let html = await this.testFilePromise;
318+
this.logger.debug('testfile_ready', `Finished fetching ${this.testFile}`);
280319

281320
// Head injection
282321
// * Use a callback, to avoid accidental $1 substitutions via user input.
@@ -319,6 +358,7 @@ class ControlServer {
319358
if (browser) {
320359
browser.clientIdleActive = performance.now();
321360
browser.logger.debug('browser_connected', `${browser.getDisplayName()} connected! Serving test file.`);
361+
this.eventbus.emit('clientonline', { clientId });
322362
} else {
323363
this.logger.debug('respond_static_testfile', clientId);
324364
}
@@ -355,6 +395,7 @@ class ControlServer {
355395
const clientId = url.searchParams.get('qtap_clientId');
356396
const browser = this.browsers.get(clientId);
357397
if (browser) {
398+
this.eventbus.emit('clientsample', { clientId, line: body.trimEnd().split('\n').pop() });
358399
const now = performance.now();
359400
browser.logger.debug('browser_tap_received',
360401
`+${util.humanSeconds(now - browser.clientIdleActive)}s`,
@@ -369,66 +410,6 @@ class ControlServer {
369410
});
370411
resp.writeHead(204);
371412
resp.end();
372-
373-
// TODO: Pipe to one of two options, based on --reporter:
374-
// - [tap | default in piped and CI?]: tap-parser + some kind of renumbering or prefixing.
375-
// client_1> ok 40 foo > bar
376-
// out> ok 1 - qtap > Firefox (client_1) connected! Running test/index.html.
377-
// out> ok 42 - foo > bar
378-
// -->
379-
// client_1> ok 40 foo > bar
380-
// client_2> ok 40 foo > bar
381-
// out> ok 1 - qtap > Firefox (client_1) connected! Running test/index.html.
382-
// out> ok 2 - qtap > Chromium (client_2) connected! Running test/index.html.
383-
// out> ok 81 - foo > bar [Firefox client_1]
384-
// out> ok 82 - foo > bar [Chromium client_2]
385-
// - [minimal|default in interactive mode]
386-
// out> Testing /test/index.html
387-
// out>
388-
// out> Firefox : SPINNER [blue]
389-
// out> Running test 40.
390-
// out> [Chromium] : [grey] [star] [grey] Launching...
391-
// out> [Safari] : [grey] [star] [grey] Launching...
392-
// -->
393-
// out> Testing /test/index.html
394-
// out>
395-
// out> [Firefox client_1]: ✔ [green] Completed 123 tests in 42ms.
396-
// out> [Chromium client2]: [blue*spinner] Running test 40.
397-
// out> [Safari client_3] [grey] [star] [grey] Launching...
398-
// -->
399-
// out> Testing /test/index.html
400-
// out>
401-
// out> not ok 40 foo > bar # Chromium client_2
402-
// out> ---
403-
// out> message: failed
404-
// out> actual : false
405-
// out> expected: true
406-
// out> stack: |
407-
// out> @/example.js:46:12
408-
// out> ...
409-
// out>
410-
// out> [Firefox client_1]: ✔ [green] Completed 123 tests in 42ms.
411-
// out> [Chromium client_2]: ✘ [red] 2 failures.
412-
//
413-
// If minimal is selected explicilty in piped/non-interactive/CI mode,
414-
// then it will have no spinners, and also lines won't overwrite each other.
415-
// Test counting will be disabled along with the spinner so instead we'll print:
416-
// out> Firefox client_1: Launching...
417-
// out> Firefox client_1: Running tests... [= instead of spinner/counter]
418-
// out> Firefox client_1: Completed 123 tets in 42ms.
419-
420-
// "▓", "▒", "░" // noise, 100
421-
// "㊂", "㊀", "㊁" // toggle10, 100
422-
// await new Promise(r=>setTimeout(r,100)); process.stdout.write('\r' + frames[i % frames.length] + ' ');
423-
// writable.isTTY
424-
// !process.env.CI
425-
426-
// Default: TAP where each browser is 1 virtual test in case of success.
427-
// Verbose: TAP forwarded, test names prepended with [browsername].
428-
// Failures are shown either way, with prepended names.
429-
// TODO: On "runEnd", report runtime
430-
// Default: No-op, as overall TAP line as single test (above) can contain runtime
431-
// Verbose: Output comment indicatinh browser done, and test runtime.
432413
}
433414

434415
/**

‎test/fail-and-timeout.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
delay(100, () => {
1212
console.log("not ok 2 \u001b[31mBaz > hello fail\u001b[39m");
13-
console.log(" ---\n message: \"y\"\n severity: failed\n actual : {\n \"foo\": \"50\",\n \"quux\": \"62\"\n}\n expected: {\n \"foo\": \"50\",\n \"quux\": \"70\"\n}\n stack: |\n @https://demo.localhost:4000/test/fail.html:21:12\n ...");
13+
console.log(" ---\n message: \"y\"\n severity: failed\n actual : {\n \"foo\": \"50\",\n \"quux\": \"62\"\n }\n expected: {\n \"foo\": \"50\",\n \"quux\": \"70\"\n }\n stack: |\n @https://demo.localhost:4000/example.html:21:12\n ...");
1414

1515
delay(100, () => {
1616
console.log('ok 3 Baz > another thing');

‎test/fail-and-uncaught.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
console.log('TAP version 13');
55
console.log('ok 1 Foo bar');
66
console.log("not ok 2 \u001b[31mBaz > hello fail\u001b[39m");
7-
console.log(" ---\n message: \"y\"\n severity: failed\n actual : {\n \"foo\": \"50\",\n \"quux\": \"62\"\n}\n expected: {\n \"foo\": \"50\",\n \"quux\": \"70\"\n}\n stack: |\n @https://demo.localhost:4000/test/fail.html:21:12\n ...");
7+
console.log(" ---\n message: \"y\"\n severity: failed\n actual : {\n \"foo\": \"50\",\n \"quux\": \"62\"\n }\n expected: {\n \"foo\": \"50\",\n \"quux\": \"70\"\n }\n stack: |\n @https://demo.localhost:4000/example.html:21:12\n ...");
88
</script>
99
<script>
1010
setTimeout(() => {

‎test/fail.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
console.log('TAP version 13');
55
console.log('ok 1 Foo bar');
66
console.log("not ok 2 \u001b[31mBaz > hello fail\u001b[39m");
7-
console.log(" ---\n message: \"y\"\n severity: failed\n actual : {\n \"foo\": \"50\",\n \"quux\": \"62\"\n}\n expected: {\n \"foo\": \"50\",\n \"quux\": \"70\"\n}\n stack: |\n @https://demo.localhost:4000/test/fail.html:21:12\n ...");
7+
console.log(" ---\n message: \"y\"\n severity: failed\n actual : {\n \"foo\": \"50\",\n \"quux\": \"62\"\n }\n expected: {\n \"foo\": \"50\",\n \"quux\": \"70\"\n }\n stack: |\n @https://demo.localhost:4000/example.html:21:12\n ...");
88
console.log('ok 3 Quux');
99
console.log('1..3');
1010
</script>

‎test/fixtures/qtap.config.js

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
export default {
2+
browsers: {
3+
noop_true () {
4+
},
5+
async noop_false (url, signals) {
6+
await new Promise((resolve, reject) => {
7+
setTimeout(() => {
8+
reject(new Error('Burp'));
9+
}, 3000);
10+
signals.browser.addEventListener('abort', () => {
11+
reject(new Error('Boo'));
12+
});
13+
});
14+
}
15+
}
16+
};

‎test/qtap.test.js

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import path from 'node:path';
2+
import { fileURLToPath } from 'url';
3+
import util from 'node:util';
4+
5+
import qtap from '../src/qtap.js';
6+
// import { stripAsciEscapes } from '../src/util.js';
7+
8+
const __filename = fileURLToPath(import.meta.url);
9+
const __dirname = path.dirname(__filename);
10+
const root = path.join(__dirname, '..');
11+
const options = {
12+
root,
13+
timeout: 2,
14+
verbose: !!process.env.CI,
15+
// verbose: true, // debugging
16+
printDebug: (str) => { console.error('# ' + str); }
17+
};
18+
19+
function debugReporter (eventbus) {
20+
const steps = [];
21+
eventbus.on('clientcreate', (event) => steps.push(`clientcreate: running ${event.testFile}`));
22+
eventbus.on('clientonline', () => steps.push('clientonline'));
23+
eventbus.on('clientbail', (event) => steps.push(`clientbail: ${event.reason}`));
24+
eventbus.on('clientresult', (event) => {
25+
steps.push(`clientresult: ${util.inspect(event.result, { colors: false })}`);
26+
});
27+
return steps;
28+
}
29+
30+
QUnit.module('qtap', function () {
31+
QUnit.test.each('run', {
32+
pass: {
33+
files: 'test/pass.html',
34+
options,
35+
expected: [
36+
'clientcreate: running test/pass.html',
37+
'clientonline',
38+
'clientresult: { ok: true, total: 4, passed: 4, failed: 0 }',
39+
],
40+
exitCode: 0
41+
},
42+
fail: {
43+
files: 'test/fail.html',
44+
options,
45+
expected: [
46+
'clientcreate: running test/fail.html',
47+
'clientonline',
48+
'clientresult: { ok: false, total: 3, passed: 2, failed: 1 }',
49+
],
50+
exitCode: 1
51+
},
52+
bail: {
53+
files: 'test/bail.html',
54+
options,
55+
expected: [
56+
'clientcreate: running test/bail.html',
57+
'clientonline',
58+
'clientbail: Need more cowbell.',
59+
],
60+
exitCode: 1
61+
},
62+
slow: {
63+
files: 'test/slow.html',
64+
options,
65+
expected: [
66+
'clientcreate: running test/slow.html',
67+
'clientonline',
68+
'clientresult: { ok: true, total: 4, passed: 4, failed: 0 }',
69+
],
70+
exitCode: 0
71+
},
72+
timeout: {
73+
files: 'test/timeout.html',
74+
options,
75+
expected: [
76+
'clientcreate: running test/timeout.html',
77+
'clientonline',
78+
'clientbail: Browser idle for 2s',
79+
],
80+
exitCode: 1
81+
},
82+
uncaughtEarly: {
83+
files: 'test/uncaught-early.html',
84+
options,
85+
expected: [
86+
'clientcreate: running test/uncaught-early.html',
87+
'clientonline',
88+
'clientbail: Browser idle for 2s',
89+
],
90+
exitCode: 1
91+
},
92+
uncaughtMid: {
93+
files: 'test/uncaught-mid.html',
94+
options,
95+
expected: [
96+
'clientcreate: running test/uncaught-mid.html',
97+
'clientonline',
98+
'clientbail: Browser idle for 2s',
99+
],
100+
exitCode: 1
101+
},
102+
uncaughtLate: {
103+
files: 'test/uncaught-late.html',
104+
options,
105+
expected: [
106+
'clientcreate: running test/uncaught-late.html',
107+
'clientonline',
108+
'clientresult: { ok: true, total: 4, passed: 4, failed: 0 }',
109+
],
110+
exitCode: 0
111+
},
112+
failAndTimeout: {
113+
files: 'test/fail-and-timeout.html',
114+
options,
115+
expected: [
116+
'clientcreate: running test/fail-and-timeout.html',
117+
'clientonline',
118+
'clientbail: Browser idle for 2s',
119+
],
120+
exitCode: 1
121+
},
122+
failAndUncaught: {
123+
files: 'test/fail-and-uncaught.html',
124+
options,
125+
expected: [
126+
'clientcreate: running test/fail-and-uncaught.html',
127+
'clientonline',
128+
'clientbail: Browser idle for 2s',
129+
],
130+
exitCode: 1
131+
},
132+
}, async function (assert, params) {
133+
assert.timeout(10_000);
134+
135+
const run = qtap.run(
136+
'firefox',
137+
params.files,
138+
params.options
139+
);
140+
const steps = debugReporter(run);
141+
142+
const result = await new Promise((resolve, reject) => {
143+
run.on('finish', resolve);
144+
run.on('error', reject);
145+
});
146+
147+
assert.deepEqual(steps, params.expected, 'Output');
148+
assert.deepEqual(result.exitCode, params.exitCode, 'Exit code');
149+
});
150+
});

‎test/slow.html

+6-8
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,15 @@
44
function delay(ms, fn) {
55
setTimeout(fn, ms);
66
}
7-
delay(1000, () => {
7+
delay(500, () => {
88
console.log('TAP version 13');
9-
delay(2000, () => {
9+
delay(500, () => {
1010
console.log('ok 1 Foo');
11+
console.log('ok 2 Bar > this thing');
1112
delay(2000, () => {
12-
console.log('ok 2 Bar > this thing');
13-
delay(2000, () => {
14-
console.log('ok 3 Bar > another thing');
15-
console.log('ok 4 Quux');
16-
console.log('1..4');
17-
});
13+
console.log('ok 3 Bar > another thing');
14+
console.log('ok 4 Quux');
15+
console.log('1..4');
1816
});
1917
});
2018
});

‎test/timeout.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
function delay(ms, fn) {
66
setTimeout(fn, ms);
77
}
8-
delay(1000, () => {
8+
delay(500, () => {
99
console.log('ok 1 Foo bar');
1010

11-
delay(2100, () => {
11+
delay(500, () => {
1212
console.log('ok 2 Baz > this thing');
1313

1414
delay(3200, () => {

0 commit comments

Comments
 (0)
Please sign in to comment.