Skip to content

Commit

Permalink
test_runner: add snapshot testing
Browse files Browse the repository at this point in the history
This commit adds a t.assert.snapshot() method that implements
snapshot testing.
  • Loading branch information
cjihrig committed May 26, 2024
1 parent b7faaaa commit 67a15b1
Show file tree
Hide file tree
Showing 11 changed files with 628 additions and 4 deletions.
1 change: 1 addition & 0 deletions lib/internal/test_runner/harness.js
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ function setup(root) {
counters: null,
shouldColorizeTestFiles: false,
teardown: exitHandler,
snapshotManager: null,
};
root.harness.resetCounters();
root.startTime = hrtime();
Expand Down
234 changes: 234 additions & 0 deletions lib/internal/test_runner/snapshot.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,234 @@
'use strict';
const {
ArrayPrototypeJoin,
ArrayPrototypeMap,
ArrayPrototypeSort,
JSONStringify,
ObjectKeys,
SafeMap,
String,
StringPrototypeReplaceAll,
} = primordials;
const {
codes: {
ERR_INVALID_STATE,
},
} = require('internal/errors');
const { emitExperimentalWarning, kEmptyObject } = require('internal/util');
let debug = require('internal/util/debuglog').debuglog('test_runner', (fn) => {
debug = fn;
});
const {
validateArray,
validateFunction,
validateObject,
} = require('internal/validators');
const { strictEqual } = require('assert');
const { mkdirSync, readFileSync, writeFileSync } = require('fs');
const { dirname } = require('path');
const { createContext, runInContext } = require('vm');
const kExperimentalWarning = 'Snapshot testing';
const kMissingSnapshotTip = 'Missing snapshots can be generated by rerunning ' +
'the command with the --test-update-snapshots flag.';
const defaultSerializers = [
(value) => { return JSONStringify(value, null, 2); },
];

function defaultResolveSnapshotPath(testPath) {
if (typeof testPath !== 'string') {
return testPath;
}

return `${testPath}.snapshot`;
}

let resolveSnapshotPathFn = defaultResolveSnapshotPath;
let serializerFns = defaultSerializers;

function setResolveSnapshotPath(fn) {
emitExperimentalWarning(kExperimentalWarning);
validateFunction(fn, 'fn');
resolveSnapshotPathFn = fn;
}

function setDefaultSerializers(serializers) {
emitExperimentalWarning(kExperimentalWarning);
validateFunctionArray(serializers, 'serializers');
serializerFns = ArrayPrototypeMap(serializers, (fn) => {
return fn;
});
}

class SnapshotManager {
constructor(entryFile, updateSnapshots) {
this.entryFile = entryFile;
this.snapshotFile = undefined;
this.snapshots = { __proto__: null };
this.nameCounts = new SafeMap();
// A manager instance will only read or write snapshot files based on the
// updateSnapshots argument.
this.loaded = updateSnapshots;
this.updateSnapshots = updateSnapshots;
}

resolveSnapshotFile() {
if (this.snapshotFile === undefined) {
const resolved = resolveSnapshotPathFn(this.entryFile);

if (typeof resolved !== 'string') {
const err = new ERR_INVALID_STATE('Invalid snapshot filename.');
err.filename = resolved;
throw err;
}

this.snapshotFile = resolved;
}
}

serialize(input, serializers = serializerFns) {
try {
let value = input;

for (let i = 0; i < serializers.length; ++i) {
const fn = serializers[i];
value = fn(value);
}

return `\n${templateEscape(value)}\n`;
} catch (err) {
const error = new ERR_INVALID_STATE(
'The provided serializers did not generate a string.',
);
error.input = input;
error.cause = err;
throw error;
}
}

getSnapshot(id) {
if (!(id in this.snapshots)) {
const err = new ERR_INVALID_STATE(`Snapshot '${id}' not found in ` +
`'${this.snapshotFile}.' ${kMissingSnapshotTip}`);
err.snapshot = id;
err.filename = this.snapshotFile;
throw err;
}

return this.snapshots[id];
}

setSnapshot(id, value) {
this.snapshots[templateEscape(id)] = value;
}

nextId(name) {
const count = this.nameCounts.get(name) ?? 1;

this.nameCounts.set(name, count + 1);
return `${name} ${count}`;
}

readSnapshotFile() {
if (this.loaded) {
debug('skipping read of snapshot file');
return;
}

try {
const source = readFileSync(this.snapshotFile, 'utf8');
const context = { __proto__: null, exports: { __proto__: null } };

createContext(context);
runInContext(source, context);

if (context.exports === null || typeof context.exports !== 'object') {
throw new ERR_INVALID_STATE(
`Malformed snapshot file '${this.snapshotFile}'.`,
);
}

this.snapshots = context.exports;
this.loaded = true;
} catch (err) {
let msg = `Cannot read snapshot file '${this.snapshotFile}.'`;

if (err?.code === 'ENOENT') {
msg += ` ${kMissingSnapshotTip}`;
}

const error = new ERR_INVALID_STATE(msg);
error.cause = err;
error.filename = this.snapshotFile;
throw error;
}
}

writeSnapshotFile() {
if (!this.updateSnapshots) {
debug('skipping write of snapshot file');
return;
}

const keys = ArrayPrototypeSort(ObjectKeys(this.snapshots));
const snapshotStrings = ArrayPrototypeMap(keys, (key) => {
return `exports[\`${key}\`] = \`${this.snapshots[key]}\`;\n`;
});
const output = ArrayPrototypeJoin(snapshotStrings, '\n');
mkdirSync(dirname(this.snapshotFile), { __proto__: null, recursive: true });
writeFileSync(this.snapshotFile, output, 'utf8');
}

createAssert() {
const manager = this;

return function snapshotAssertion(actual, options = kEmptyObject) {
emitExperimentalWarning(kExperimentalWarning);
// Resolve the snapshot file here so that any resolution errors are
// surfaced as early as possible.
manager.resolveSnapshotFile();

const { fullName } = this;
const id = manager.nextId(fullName);

validateObject(options, 'options');

const {
serializers = serializerFns,
} = options;

validateFunctionArray(serializers, 'options.serializers');

const value = manager.serialize(actual, serializers);

if (manager.updateSnapshots) {
manager.setSnapshot(id, value);
} else {
manager.readSnapshotFile();
strictEqual(value, manager.getSnapshot(id));
}
};
}
}

function validateFunctionArray(fns, name) {
validateArray(fns, name);
for (let i = 0; i < fns.length; ++i) {
validateFunction(fns[i], `${name}[${i}]`);
}
}

function templateEscape(str) {
let result = String(str);
result = StringPrototypeReplaceAll(result, '\\', '\\\\');
result = StringPrototypeReplaceAll(result, '`', '\\`');
result = StringPrototypeReplaceAll(result, '${', '\\${');
return result;
}

module.exports = {
SnapshotManager,
defaultResolveSnapshotPath, // Exported for testing only.
defaultSerializers, // Exported for testing only.
setDefaultSerializers,
setResolveSnapshotPath,
};
17 changes: 13 additions & 4 deletions lib/internal/test_runner/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ const {
testNamePatterns,
testSkipPatterns,
testOnlyFlag,
updateSnapshots,
} = parseCommandLine();
let kResistStopPropagation;
let assertObj;
Expand All @@ -102,11 +103,10 @@ function lazyFindSourceMap(file) {
return findSourceMap(file);
}

function lazyAssertObject() {
function lazyAssertObject(harness) {
if (assertObj === undefined) {
assertObj = new SafeMap();
const assert = require('assert');

const methodsToCopy = [
'deepEqual',
'deepStrictEqual',
Expand All @@ -129,6 +129,13 @@ function lazyAssertObject() {
for (let i = 0; i < methodsToCopy.length; i++) {
assertObj.set(methodsToCopy[i], assert[methodsToCopy[i]]);
}

const { getOptionValue } = require('internal/options');
if (getOptionValue('--experimental-test-snapshots')) {
const { SnapshotManager } = require('internal/test_runner/snapshot');
harness.snapshotManager = new SnapshotManager(kFilename, updateSnapshots);
assertObj.set('snapshot', harness.snapshotManager.createAssert());
}
}
return assertObj;
}
Expand Down Expand Up @@ -248,7 +255,7 @@ class TestContext {
get assert() {
if (this.#assert === undefined) {
const { plan } = this.#test;
const map = lazyAssertObject();
const map = lazyAssertObject(this.#test.root.harness);
const assert = { __proto__: null };

this.#assert = assert;
Expand All @@ -257,7 +264,7 @@ class TestContext {
if (plan !== null) {
plan.actual++;
}
return ReflectApply(method, assert, args);
return ReflectApply(method, this, args);
};
});
}
Expand Down Expand Up @@ -960,6 +967,7 @@ class Test extends AsyncResource {

// Call this harness.coverage() before collecting diagnostics, since failure to collect coverage is a diagnostic.
const coverage = harness.coverage();
harness.snapshotManager?.writeSnapshotFile();
for (let i = 0; i < diagnostics.length; i++) {
reporter.diagnostic(nesting, loc, diagnostics[i]);
}
Expand All @@ -980,6 +988,7 @@ class Test extends AsyncResource {
if (harness.watching) {
this.reported = false;
harness.resetCounters();
assertObj = undefined;
} else {
reporter.end();
}
Expand Down
2 changes: 2 additions & 0 deletions lib/internal/test_runner/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ function parseCommandLine() {
const coverage = getOptionValue('--experimental-test-coverage');
const forceExit = getOptionValue('--test-force-exit');
const sourceMaps = getOptionValue('--enable-source-maps');
const updateSnapshots = getOptionValue('--test-update-snapshots');
const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
let destinations;
Expand Down Expand Up @@ -255,6 +256,7 @@ function parseCommandLine() {
testOnlyFlag,
testNamePatterns,
testSkipPatterns,
updateSnapshots,
reporters,
destinations,
};
Expand Down
27 changes: 27 additions & 0 deletions lib/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const {

const { test, suite, before, after, beforeEach, afterEach } = require('internal/test_runner/harness');
const { run } = require('internal/test_runner/runner');
const { getOptionValue } = require('internal/options');

module.exports = test;
ObjectAssign(module.exports, {
Expand Down Expand Up @@ -37,3 +38,29 @@ ObjectDefineProperty(module.exports, 'mock', {
return lazyMock;
},
});

if (getOptionValue('--experimental-test-snapshots')) {
let lazySnapshot;

ObjectDefineProperty(module.exports, 'snapshot', {
__proto__: null,
configurable: true,
enumerable: true,
get() {
if (lazySnapshot === undefined) {
const {
setDefaultSerializers,
setResolveSnapshotPath,
} = require('internal/test_runner/snapshot');

lazySnapshot = {
__proto__: null,
setDefaultSerializers,
setResolveSnapshotPath,
};
}

return lazySnapshot;
},
});
}
6 changes: 6 additions & 0 deletions src/node_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -623,12 +623,18 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
AddOption("--test-timeout",
"specify test runner timeout",
&EnvironmentOptions::test_runner_timeout);
AddOption("--test-update-snapshots",
"regenerate test snapshots",
&EnvironmentOptions::test_runner_update_snapshots);
AddOption("--experimental-test-coverage",
"enable code coverage in the test runner",
&EnvironmentOptions::test_runner_coverage);
AddOption("--experimental-test-module-mocks",
"enable module mocking in the test runner",
&EnvironmentOptions::test_runner_module_mocks);
AddOption("--experimental-test-snapshots",
"enable snapshot testing in the test runner",
&EnvironmentOptions::test_runner_snapshots);
AddOption("--test-name-pattern",
"run tests whose name matches this regular expression",
&EnvironmentOptions::test_name_pattern);
Expand Down
2 changes: 2 additions & 0 deletions src/node_options.h
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ class EnvironmentOptions : public Options {
bool test_runner_coverage = false;
bool test_runner_force_exit = false;
bool test_runner_module_mocks = false;
bool test_runner_snapshots = false;
bool test_runner_update_snapshots = false;
std::vector<std::string> test_name_pattern;
std::vector<std::string> test_reporter;
std::vector<std::string> test_reporter_destination;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
exports = null;
Loading

0 comments on commit 67a15b1

Please sign in to comment.