server,browsers: Promote bailout to error and fix multi file in Safari

Minor:

* browsers: Fix safari() bug when running multiple test files.
* browsers: Rename Headless Firefox => Firefox Headless.
* client: Set `window.qunit_config_reporters_html = false`.
* client: Add basic visual `<pre>` output in debug mode.
* server: Change clientId format from "client_1" to "client_S1_C1"
  to make verbose logs more useful. This makes it obvious which
  clients are for the same testFile, and thus ControlServer.

* qtap: De-duplicate browsers and test files.

Major:

* qtap,server: Streamline error handling and signal handling to avoid
  errors going unhandled, bypassing teardown, or causing
  the process to be stuck waiting for nothing.

  In particular, "Could not open <file>" is now propagated to a
  top-level error (e.g. run throws/rejects) rather than a test-level
  "bail". This means we emit "error" instead of "result" or "finish",
  and other tests are cancelled as well.

  With this in place, other uncaught errors are now handled better
  as well, such as errors from reporters.

* qtap: handle uncaught errors from reporters, which otherwise
  cause emit() to throw. Provide a dedicated EventEmitter proxy
  to improve attribution in the error message to a given reporter.
  This is limited to "on" both because its simpler that way, and
  because it protects our EventEmitter object from reporters
  tampering with "emit" or "on".

* reporter: Flesh out more of the default reporter,
  and refine the events we emit to ease the work of reporters.

  In particular, promote all bailouts to first-class errors that
  bubble up and cancel other clients. This way, reporters don't need
  to deal with formatting bailout as assert failure or or numbering
  test counts (i.e. Completed 0 tests, 0 failed.), and thus don't
  receive a "finish" event.

Author: Krinkle

API.md

@@ -155,70 +155,78 @@ async function mybrowser (url, signals) {
 }
 ```
 
-## QTap basic events
+## QTap summary events
 
-### Event: `'error'`
+These are emitted at most once for the run overall.
 
-* `error <Error|string>`
+### Event: `'clients'`
 
-### Event: `'finish'`
+The `clients` event conveys which browsers are being started, and which tests will be run. It is emitted as soon as QTap has validated the parameters. Each client is a browser process that runs one test suite. For example, if you run 2 test suites in 3 different browsers, there will be 6 clients.
 
-Summary event that is ideal for when you run one test suite in one browser, or if you otherwise don't need a break down of results by client.
+* `event.clients {Object<string,Object>}` Keyed by clientId
+  * `clientId {string}` An identifier unique within the current qtap process (e.g. `client_123`).
+  * `testFile {string}` Relative file path or URL (e.g. `test/index.html` or `http://localhost/test/`).
+  * `browserName {string}` Browser name, as specified in config or CLI (e.g. `firefox`).
+  * `displayName {string}` Browser pretty name (e.g. "Headless Firefox").
 
-* `event.ok <boolean>` Aggregate status of each client's results. If any failed, this is false.
-* `event.exitCode <number>` Suggested exit code, 0 for success, 1 for failed.
-* `event.total <number>` Aggregated from `result` events.
-* `event.passed <number>` Aggregated from `result` events.
-* `event.failed <number>` Aggregated from `result` events.
-* `event.skips <array>` Carried from the first client that failed, or empty.
-* `event.todos <array>` Carried from the first client that failed, or empty.
-* `event.failures <array>` Carried from the first client that failed, or empty.
-* `event.bailout <false|string>` Carried from the first client that failed, or false.
+### Event: `'error'`
 
-## QTap reporter events
+* `error {Error|string}`
+
+### Event: `'finish'`
 
-A client will emit each of these events only once, except `consoleerror` which may be emitted any number of times.
+Summary event that is ideal for when you run one test suite in one browser, or if you don't need a break down of results by client.
 
-### Event: `'client'`
+* `event.ok {boolean}` Aggregate status of each client's results. If any failed, this is false.
+* `event.exitCode {number}` Suggested exit code, 0 for success, 1 for failed.
+* `event.total {number}` Aggregated from `result` events.
+* `event.passed {number}` Aggregated from `result` events.
+* `event.failed {number}` Aggregated from `result` events.
+* `event.skips {array}` Carried from the first client that failed, or empty.
+* `event.todos {array}` Carried from the first client that failed, or empty.
+* `event.failures {array}` Carried from the first client that failed, or empty.
 
-The `client` event is emitted when a client is created. A client is a dedicated browser instance that runs one test suite. For example, if you run 2 test suites in 3 different browsers, there will be 6 clients.
+## QTap client events
 
-* `event.clientId <string>` An identifier unique within the current qtap process (e.g. `client_123`).
-* `event.testFile <string>` Relative file path or URL (e.g. `test/index.html` or `http://localhost/test/`).
-* `event.browserName <string>` Browser name, as specified in config or CLI (e.g. `firefox`).
-* `event.displayName <string>` Browser pretty name, (e.g. "Headless Firefox").
+These are emitted once per client, except `consoleerror` and `assert` which may be emitted many times by a client during a test run.
 
 ### Event: `'online'`
 
-The `online` event is emitted when a browser has successfully started and opened the test file. If a browser fails to connect, a `bail` event is emitted instead.
+The `online` event is emitted when a browser has successfully started and opened the test file. If a browser fails to connect or a test run bailed out, then an `error` event is emitted instead.
 
-* `event.clientId <string>`
+* `event.clientId {string}`
+* `event.testFile {string}`
+* `event.browserName {string}`
+* `event.displayName {string}`
 
 ### Event: `'result'`
 
-The `result` event is emitted when a browser has completed a test run. This is mutually exclusive with the `bail` event.
+The `result` event is emitted when a browser has completed a test run.
 
-* `event.clientId <string>`
-* `event.ok <boolean>`
-* `event.total <number>`
-* `event.passed <number>`
-* `event.failed <number>`
-* `event.skips <array>` Details about skipped tests (count as passed).
-* `event.todos <array>` Details about todo tests (count as passed).
-* `event.failures <array>` Details about failed tests.
+* `event.clientId {string}`
+* `event.ok {boolean}`
+* `event.total {number}`
+* `event.passed {number}`
+* `event.failed {number}`
+* `event.skips {array}` Details about skipped tests (count as passed).
+* `event.todos {array}` Details about todo tests (count as passed).
+* `event.failures {array}` Details about failed tests.
 
-### Event: `'bail'`
+### Event: `'consoleerror'`
 
-The `bail` event is emitted when a browser was unable to start or complete a test run.
+The `consoleerror` event relays any warning or error messages from the browser console. These are for debug purposes only, and do not indicate that a test has failed. A complete and successful test run, may nonetheless print warnings or errors to the console.
 
-* `event.clientId <string>`
-* `event.reason <string>`
+It is recommended that reporters only display console errors if a test run failed (i.e. there was a failed test result, or an uncaught error).
 
-### Event: `'consoleerror'`
+* `event.clientId {string}`
+* `event.message {string}`
 
-The `consoleerror` event relays any warning or error messages from the browser console. These are for debug purposes only, and do not indicate that a test has failed. A complete and successful test run, may nonetheless print warnings or errors to the console.
+### Event: `'assert'`
 
-It is recommended that reporters only display console errors if a test run failed (i.e. there was a failed test result, or the cilent bailed).
+The `assert` event describes a single test result (whether passing or failing). This can be used by reporters to indicate activity, display the name of a test in real-time, or to convey failures early.
 
-* `event.clientId <string>`
-* `event.message <string>`
+* `event.clientId {string}`
+* `event.result {Object}`
+  * `ok {boolean}`
+  * `fullname {string}`
+  * `diag {undefined|Object}`

ARCHITECTURE.md

@@ -436,7 +436,7 @@ Safari has long resisted the temptation to offer a reasonable command-line inter
   {"value":null}
   ```
 
-  This addresses all previous concerns, and seems to be the best as of 2025. The only downside is that it requires a bit more code to setup (find available port, and perform various HTTP requests).
+  This addresses all previous concerns, and seems to work best as of 2025. The only downside is that it requires more code to set up (find available port, and perform various HTTP requests).
 
   - https://webkit.org/blog/6900/webdriver-support-in-safari-10/
   - https://developer.apple.com/documentation/webkit/macos-webdriver-commands-for-safari-12-and-later

bin/qtap.js

@@ -1,6 +1,8 @@
 #!/usr/bin/env node
 'use strict';
 
+import util from 'node:util';
+
 import { program, InvalidArgumentError } from 'commander';
 import qtap from '../src/qtap.js';
 
@@ -90,7 +92,31 @@ if (opts.version) {
     });
     process.exit(result.exitCode);
   } catch (e) {
-    console.error(e);
+    if (e instanceof qtap.QTapError && e.qtapClient) {
+      console.log(
+        '\n'
+        + util.styleText('bgRedBright',
+          util.styleText('redBright', '__')
+          + util.styleText(['whiteBright', 'bold'], 'BAILED')
+          + util.styleText('redBright', '__')
+        )
+        + '\n'
+      );
+      console.error(util.styleText('grey',
+        `Bail out from ${e.qtapClient.testFile} in ${e.qtapClient.browser}:`
+      ));
+      // Omit internal stack trace, not useful here
+      console.error(util.styleText('bold',
+        e.message
+      ));
+    } else {
+      if (e instanceof qtap.QTapError) {
+        console.error(e.toString());
+      } else {
+        // Print including full stack trace
+        console.error(e);
+      }
+    }
     process.exit(1);
   }
 }

eslint.config.js

@@ -25,7 +25,7 @@ export default [
     }
   },
   {
-    files: ['test/*.js'],
+    files: ['test/**/*.js'],
     languageOptions: {
       globals: {
         QUnit: 'readonly'

src/browsers.js

@@ -118,7 +118,7 @@ async function firefox (url, signals, logger, debugMode) {
   const profileDir = LocalBrowser.makeTempDir(signals, logger);
   const args = [url, '-profile', profileDir, '-no-remote', '-wait-for-browser'];
   if (!debugMode) {
-    firefox.displayName = 'Headless Firefox';
+    firefox.displayName = 'Firefox Headless';
     args.push('-headless');
   }
 
@@ -164,7 +164,7 @@ firefox.displayName = 'Firefox';
 function makeChromium (displayName, getPaths) {
   /** @type {Browser} - https://github.com/microsoft/TypeScript/issues/22063 */
   const chromium = async function (url, signals, logger, debugMode) {
-    chromium.displayName = debugMode ? displayName : `Headless ${displayName}`;
+    chromium.displayName = debugMode ? displayName : `${displayName} Headless`;
     // https://github.com/GoogleChrome/chrome-launcher/blob/main/docs/chrome-flags-for-tools.md
     const dataDir = LocalBrowser.makeTempDir(signals, logger);
     const args = [

src/client.cjs

@@ -4,10 +4,12 @@
 function qtapClientHead () {
   // Support QUnit 2.24+: Enable TAP reporter, declaratively.
   window.qunit_config_reporters_tap = true;
+  window.qunit_config_reporters_html = false;
 
   // Cache references to original methods, to avoid getting trapped by mocks (e.g. Sinon)
   var setTimeout = window.setTimeout;
   var XMLHttpRequest = window.XMLHttpRequest;
+  var createTextNode = document.createTextNode && document.createTextNode.bind && document.createTextNode.bind(document);
 
   // Support IE 9: console.log.apply is undefined.
   // Don't bother with Function.apply.call. Skip super call instead.
@@ -67,6 +69,7 @@ function qtapClientHead () {
   function createBufferedWrite (url) {
     var buffer = '';
     var isSending = false;
+    var debugElement = false;
     function send () {
       var body = buffer;
       buffer = '';
@@ -81,8 +84,16 @@ function qtapClientHead () {
       };
       xhr.open('POST', url, true);
       xhr.send(body);
+
+      // Optimization: Only check this once, during the first send
+      if (debugElement === false) {
+        debugElement = document.getElementById('__qtap_debug_element') || null;
+      }
+      if (debugElement) {
+        debugElement.appendChild(createTextNode(body));
+      }
     }
-    return function write (str) {
+    return function writeTap (str) {
       buffer += str + '\n';
       if (!isSending) {
         isSending = true;