fix: ensure reporter errors can propagate (#4111)

* feat: allow reporters to error

* fix(reporter): reject handle for consistent error processing

axe.d.ts

@@ -371,7 +371,8 @@ declare namespace axe {
   type AxeReporter<T = unknown> = (
     rawResults: RawResult[],
     option: RunOptions,
-    callback: (report: T) => void
+    resolve: (report: T) => void,
+    reject: (error: Error) => void
   ) => void;
 
   interface VirtualNode {

lib/core/public/finish-run.js

@@ -52,8 +52,8 @@ function getMergedFrameSpecs({
 }
 
 function createReport(results, options) {
-  return new Promise(resolve => {
+  return new Promise((resolve, reject) => {
     const reporter = getReporter(options.reporter);
-    reporter(results, options, resolve);
+    reporter(results, options, resolve, reject);
   });
 }

lib/core/public/run.js

@@ -41,23 +41,29 @@ export default function run(...args) {
       axe._running = false;
       teardown();
       try {
-        callback(null, results);
+        resolve(results);
       } catch (e) {
         axe.log(e);
       }
-      resolve(results);
     };
+    const wrappedReject = err => {
+      axe._running = false;
+      teardown();
+      try {
+        reject(err);
+      } catch (e) {
+        axe.log(e);
+      }
+    };
+
     if (options.performanceTimer) {
       axe.utils.performanceTimer.end();
     }
 
     try {
-      createReport(rawResults, options, respond);
+      createReport(rawResults, options, respond, wrappedReject);
     } catch (err) {
-      axe._running = false;
-      teardown();
-      callback(err);
-      reject(err);
+      wrappedReject(err);
     }
   }
 
@@ -82,14 +88,15 @@ function getPromiseHandlers(callback) {
       resolve = _resolve;
     });
   } else {
-    resolve = reject = noop;
+    resolve = result => callback(null, result);
+    reject = err => callback(err);
   }
   return { thenable, reject, resolve };
 }
 
-function createReport(rawResults, options, respond) {
+function createReport(rawResults, options, respond, reject) {
   const reporter = getReporter(options.reporter);
-  const results = reporter(rawResults, options, respond);
+  const results = reporter(rawResults, options, respond, reject);
   if (results !== undefined) {
     respond(results);
   }

test/core/public/finish-run.js

@@ -212,6 +212,40 @@ describe('axe.finishRun', function () {
       .catch(done);
   });
 
+  it('rejects with sync reporter errors', async () => {
+    axe.addReporter('throwing', () => {
+      throw new Error('Something went wrong');
+    });
+    const options = { reporter: 'throwing' };
+
+    fixture.innerHTML = '<h1>Hello world</h1>';
+    const partial = await axe.runPartial('#fixture', options);
+    try {
+      await axe.finishRun([partial], options);
+      assert.fail('Should have thrown');
+    } catch (err) {
+      assert.equal(err.message, 'Something went wrong');
+    }
+  });
+
+  it('rejects with async reporter errors', async () => {
+    axe.addReporter('throwing', (results, options, resolve, reject) => {
+      setTimeout(() => {
+        reject(new Error('Something went wrong'));
+      }, 10);
+    });
+    const options = { reporter: 'throwing' };
+
+    fixture.innerHTML = '<h1>Hello world</h1>';
+    const partial = await axe.runPartial('#fixture', options);
+    try {
+      await axe.finishRun([partial], options);
+      assert.fail('Should have thrown');
+    } catch (err) {
+      assert.equal(err.message, 'Something went wrong');
+    }
+  });
+
   describe('frames', function () {
     function createIframe(html, parent) {
       return new Promise(function (resolve) {

test/core/public/run.js

@@ -201,6 +201,28 @@ describe('axe.run', function () {
         done();
       });
     });
+
+    it('rejects with sync reporter errors', done => {
+      axe.addReporter('throwing', () => {
+        throw new Error('Something went wrong');
+      });
+      axe.run({ reporter: 'throwing' }, err => {
+        assert.equal(err.message, 'Something went wrong');
+        done();
+      });
+    });
+
+    it('rejects with async reporter errors', done => {
+      axe.addReporter('throwing', (results, options, resolve, reject) => {
+        setTimeout(() => {
+          reject(new Error('Something went wrong'));
+        }, 10);
+      });
+      axe.run({ reporter: 'throwing' }, err => {
+        assert.equal(err.message, 'Something went wrong');
+        done();
+      });
+    });
   });
 
   describe('promise result', function () {

typings/axe-core/axe-core-tests.ts

@@ -348,9 +348,10 @@ axe.configure({
 let fooReporter = (
   results: axe.RawResult[],
   options: axe.RunOptions,
-  cb: (out: 'foo') => void
+  resolve: (out: 'foo') => void,
+  reject: (err: Error) => void
 ) => {
-  cb('foo');
+  reject && resolve('foo');
 };
 
 axe.addReporter<'foo'>('foo', fooReporter, true);