From 9b88f46896add809feab84c28b6c4d4248d846bd Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:54:13 +0900 Subject: [PATCH 1/2] fix(benchmark): make it fair --- benchmarks/_util/index.js | 53 ++++++++++++++++++ benchmarks/benchmark.js | 104 +++++++++++++---------------------- benchmarks/post-benchmark.js | 88 +++++++---------------------- benchmarks/server.js | 10 +++- 4 files changed, 119 insertions(+), 136 deletions(-) create mode 100644 benchmarks/_util/index.js diff --git a/benchmarks/_util/index.js b/benchmarks/_util/index.js new file mode 100644 index 00000000000..75f903530ca --- /dev/null +++ b/benchmarks/_util/index.js @@ -0,0 +1,53 @@ +'use strict' + +const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100 + +function makeParallelRequests (cb) { + const promises = new Array(parallelRequests) + for (let i = 0; i < parallelRequests; ++i) { + promises[i] = new Promise(cb) + } + return Promise.all(promises) +} + +function printResults (results) { + // Sort results by least performant first, then compare relative performances and also printing padding + let last + + const rows = Object.entries(results) + // If any failed, put on the top of the list, otherwise order by mean, ascending + .sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean)) + .map(([name, result]) => { + if (!result.success) { + return { + Tests: name, + Samples: result.size, + Result: 'Errored', + Tolerance: 'N/A', + 'Difference with Slowest': 'N/A' + } + } + + // Calculate throughput and relative performance + const { size, mean, standardError } = result + const relative = last !== 0 ? (last / mean - 1) * 100 : 0 + + // Save the slowest for relative comparison + if (typeof last === 'undefined') { + last = mean + } + + return { + Tests: name, + Samples: size, + Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`, + Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`, + 'Difference with slowest': + relative > 0 ? `+ ${relative.toFixed(2)} %` : '-' + } + }) + + return console.table(rows) +} + +module.exports = { makeParallelRequests, printResults } diff --git a/benchmarks/benchmark.js b/benchmarks/benchmark.js index c48c00926a1..7e922015225 100644 --- a/benchmarks/benchmark.js +++ b/benchmarks/benchmark.js @@ -8,20 +8,20 @@ const { isMainThread } = require('node:worker_threads') const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..') +const { makeParallelRequests, printResults } = require('./_util') + let nodeFetch const axios = require('axios') let superagent let got -const util = require('node:util') -const _request = require('request') -const request = util.promisify(_request) +const { promisify } = require('node:util') +const request = promisify(require('request')) const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1 const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3 const connections = parseInt(process.env.CONNECTIONS, 10) || 50 const pipelining = parseInt(process.env.PIPELINING, 10) || 10 -const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100 const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0 const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0 const dest = {} @@ -136,53 +136,6 @@ class SimpleRequest { } } -function makeParallelRequests (cb) { - const promises = new Array(parallelRequests) - for (let i = 0; i < parallelRequests; ++i) { - promises[i] = new Promise(cb) - } - return Promise.all(promises) -} - -function printResults (results) { - // Sort results by least performant first, then compare relative performances and also printing padding - let last - - const rows = Object.entries(results) - // If any failed, put on the top of the list, otherwise order by mean, ascending - .sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean)) - .map(([name, result]) => { - if (!result.success) { - return { - Tests: name, - Samples: result.size, - Result: 'Errored', - Tolerance: 'N/A', - 'Difference with Slowest': 'N/A' - } - } - - // Calculate throughput and relative performance - const { size, mean, standardError } = result - const relative = last !== 0 ? (last / mean - 1) * 100 : 0 - - // Save the slowest for relative comparison - if (typeof last === 'undefined') { - last = mean - } - - return { - Tests: name, - Samples: size, - Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`, - Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`, - 'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-' - } - }) - - return console.table(rows) -} - const experiments = { 'http - no keepalive' () { return makeParallelRequests(resolve => { @@ -217,8 +170,8 @@ const experiments = { 'undici - pipeline' () { return makeParallelRequests(resolve => { dispatcher - .pipeline(undiciOptions, data => { - return data.body + .pipeline(undiciOptions, ({ body }) => { + return body }) .end() .pipe( @@ -288,9 +241,15 @@ if (process.env.PORT) { }) } + const axiosOptions = { + url: dest.url, + method: 'GET', + responseType: 'stream', + httpAgent: axiosAgent + } experiments.axios = () => { return makeParallelRequests(resolve => { - axios.get(dest.url, { responseType: 'stream', httpAgent: axiosAgent }).then(res => { + axios.request(axiosOptions).then(res => { res.data.pipe(new Writable({ write (chunk, encoding, callback) { callback() @@ -300,26 +259,37 @@ if (process.env.PORT) { }) } + const gotOptions = { + url: dest.url, + method: 'GET', + agent: { + http: gotAgent + }, + // avoid body processing + isStream: true + } experiments.got = () => { return makeParallelRequests(resolve => { - got.get(dest.url, { agent: { http: gotAgent } }).then(res => { - res.pipe(new Writable({ - write (chunk, encoding, callback) { - callback() - } - })).on('finish', resolve) - }).catch(console.log) + got(gotOptions).pipe(new Writable({ + write (chunk, encoding, callback) { + callback() + } + })).on('finish', resolve) }) } + const requestOptions = { + url: dest.url, + method: 'GET', + agent: requestAgent, + // avoid body toString + encoding: null + } experiments.request = () => { return makeParallelRequests(resolve => { - request(dest.url, { agent: requestAgent }).then(res => { - res.pipe(new Writable({ - write (chunk, encoding, callback) { - callback() - } - })).on('finish', resolve) + request(requestOptions).then(() => { + // already body consumed + resolve() }).catch(console.log) }) } diff --git a/benchmarks/post-benchmark.js b/benchmarks/post-benchmark.js index 80c23a67b92..041c136dcdd 100644 --- a/benchmarks/post-benchmark.js +++ b/benchmarks/post-benchmark.js @@ -5,22 +5,23 @@ const os = require('node:os') const path = require('node:path') const { Writable, Readable, pipeline } = require('node:stream') const { isMainThread } = require('node:worker_threads') + const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..') +const { makeParallelRequests, printResults } = require('./_util') + let nodeFetch const axios = require('axios') let superagent let got -const util = require('node:util') -const _request = require('request') -const request = util.promisify(_request) +const { promisify } = require('node:util') +const request = promisify(require('request')) const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1 const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3 const connections = parseInt(process.env.CONNECTIONS, 10) || 50 const pipelining = parseInt(process.env.PIPELINING, 10) || 10 -const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100 const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0 const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0 const dest = {} @@ -147,53 +148,6 @@ class SimpleRequest { } } -function makeParallelRequests (cb) { - const promises = new Array(parallelRequests) - for (let i = 0; i < parallelRequests; ++i) { - promises[i] = new Promise(cb) - } - return Promise.all(promises) -} - -function printResults (results) { - // Sort results by least performant first, then compare relative performances and also printing padding - let last - - const rows = Object.entries(results) - // If any failed, put on the top of the list, otherwise order by mean, ascending - .sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean)) - .map(([name, result]) => { - if (!result.success) { - return { - Tests: name, - Samples: result.size, - Result: 'Errored', - Tolerance: 'N/A', - 'Difference with Slowest': 'N/A' - } - } - - // Calculate throughput and relative performance - const { size, mean, standardError } = result - const relative = last !== 0 ? (last / mean - 1) * 100 : 0 - - // Save the slowest for relative comparison - if (typeof last === 'undefined') { - last = mean - } - - return { - Tests: name, - Samples: size, - Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`, - Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`, - 'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-' - } - }) - - return console.table(rows) -} - const experiments = { 'http - no keepalive' () { return makeParallelRequests(resolve => { @@ -236,8 +190,8 @@ const experiments = { this.push(null) } }), - dispatcher.pipeline(undiciOptions, data => { - return data.body + dispatcher.pipeline(undiciOptions, ({ body }) => { + return body }), new Writable({ write (chunk, encoding, callback) { @@ -341,22 +295,23 @@ if (process.env.PORT) { } const gotOptions = { + url: dest.url, method: 'POST', headers, agent: { http: gotAgent }, + // avoid body processing + isStream: true, body: data } experiments.got = () => { return makeParallelRequests(resolve => { - got(dest.url, gotOptions).then(res => { - res.pipe(new Writable({ - write (chunk, encoding, callback) { - callback() - } - })).on('finish', resolve) - }).catch(console.log) + got(gotOptions).pipe(new Writable({ + write (chunk, encoding, callback) { + callback() + } + })).on('finish', resolve) }) } @@ -365,16 +320,15 @@ if (process.env.PORT) { method: 'POST', headers, agent: requestAgent, - body: data + body: data, + // avoid body toString + encoding: null } experiments.request = () => { return makeParallelRequests(resolve => { - request(requestOptions).then(res => { - res.pipe(new Writable({ - write (chunk, encoding, callback) { - callback() - } - })).on('finish', resolve) + request(requestOptions).then(() => { + // already body consumed + resolve() }).catch(console.log) }) } diff --git a/benchmarks/server.js b/benchmarks/server.js index f35bdde97dc..a85347647ab 100644 --- a/benchmarks/server.js +++ b/benchmarks/server.js @@ -24,10 +24,16 @@ if (cluster.isPrimary) { } } else { const buf = Buffer.alloc(64 * 1024, '_') + + const headers = { + 'Content-Length': `${buf.byteLength}`, + 'Content-Type': 'text/plain; charset=UTF-8' + } let i = 0 - const server = createServer((req, res) => { + const server = createServer((_req, res) => { i++ - setTimeout(function () { + setTimeout(() => { + res.writeHead(200, headers) res.end(buf) }, timeout) }).listen(port) From 28b5934ec79d2d953f0ff50d63d8115cad670ceb Mon Sep 17 00:00:00 2001 From: tsctx <91457664+tsctx@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:29:34 +0900 Subject: [PATCH 2/2] chore: update benchmark results --- docs/README.md | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/README.md b/docs/README.md index 56f8d1a2667..de1bdcfbb97 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,24 +17,42 @@ npm i undici ## Benchmarks -The benchmark is a simple `hello world` [example](benchmarks/benchmark.js) using a +The benchmark is a simple getting data [example](benchmarks/benchmark.js) using a 50 TCP connections with a pipelining depth of 10 running on Node 20.10.0. -``` -│ Tests │ Samples │ Result │ Tolerance │ Difference with slowest │ -|─────────────────────|─────────|─────────────────|───────────|─────────────────────────| -│ got │ 45 │ 1661.71 req/sec │ ± 2.93 % │ - │ -│ node-fetch │ 20 │ 2164.81 req/sec │ ± 2.63 % │ + 30.28 % │ -│ undici - fetch │ 35 │ 2274.27 req/sec │ ± 2.70 % │ + 36.86 % │ -│ http - no keepalive │ 15 │ 2376.04 req/sec │ ± 2.99 % │ + 42.99 % │ -│ axios │ 25 │ 2612.93 req/sec │ ± 2.89 % │ + 57.24 % │ -│ request │ 40 │ 2712.19 req/sec │ ± 2.92 % │ + 63.22 % │ -│ http - keepalive │ 45 │ 4393.25 req/sec │ ± 2.86 % │ + 164.38 % │ -│ undici - pipeline │ 45 │ 5484.69 req/sec │ ± 2.87 % │ + 230.06 % │ -│ undici - request │ 55 │ 7773.98 req/sec │ ± 2.93 % │ + 367.83 % │ -│ undici - stream │ 70 │ 8425.96 req/sec │ ± 2.91 % │ + 407.07 % │ -│ undici - dispatch │ 50 │ 9488.99 req/sec │ ± 2.85 % │ + 471.04 % │ -``` +| _Tests_ | _Samples_ | _Result_ | _Tolerance_ | _Difference with slowest_ | +| :-----------------: | :-------: | :--------------: | :---------: | :-----------------------: | +| undici - fetch | 30 | 3704.43 req/sec | ± 2.95 % | - | +| http - no keepalive | 20 | 4275.30 req/sec | ± 2.60 % | + 15.41 % | +| node-fetch | 10 | 4759.42 req/sec | ± 0.87 % | + 28.48 % | +| request | 40 | 4803.37 req/sec | ± 2.77 % | + 29.67 % | +| axios | 45 | 4951.97 req/sec | ± 2.88 % | + 33.68 % | +| got | 10 | 5969.67 req/sec | ± 2.64 % | + 61.15 % | +| superagent | 10 | 9471.48 req/sec | ± 1.50 % | + 155.68 % | +| http - keepalive | 25 | 10327.49 req/sec | ± 2.95 % | + 178.79 % | +| undici - pipeline | 10 | 15053.41 req/sec | ± 1.63 % | + 306.36 % | +| undici - request | 10 | 19264.24 req/sec | ± 1.74 % | + 420.03 % | +| undici - stream | 15 | 20317.29 req/sec | ± 2.13 % | + 448.46 % | +| undici - dispatch | 10 | 24883.28 req/sec | ± 1.54 % | + 571.72 % | + +The benchmark is a simple sending data [example](benchmarks/post-benchmark.js) using a +50 TCP connections with a pipelining depth of 10 running on Node 20.10.0. + +| _Tests_ | _Samples_ | _Result_ | _Tolerance_ | _Difference with slowest_ | +| :-----------------: | :-------: | :-------------: | :---------: | :-----------------------: | +| undici - fetch | 20 | 1968.42 req/sec | ± 2.63 % | - | +| http - no keepalive | 25 | 2330.30 req/sec | ± 2.99 % | + 18.38 % | +| node-fetch | 20 | 2485.36 req/sec | ± 2.70 % | + 26.26 % | +| got | 15 | 2787.68 req/sec | ± 2.56 % | + 41.62 % | +| request | 30 | 2805.10 req/sec | ± 2.59 % | + 42.50 % | +| axios | 10 | 3040.45 req/sec | ± 1.72 % | + 54.46 % | +| superagent | 20 | 3358.29 req/sec | ± 2.51 % | + 70.61 % | +| http - keepalive | 20 | 3477.94 req/sec | ± 2.51 % | + 76.69 % | +| undici - pipeline | 25 | 3812.61 req/sec | ± 2.80 % | + 93.69 % | +| undici - request | 10 | 6067.00 req/sec | ± 0.94 % | + 208.22 % | +| undici - stream | 10 | 6391.61 req/sec | ± 1.98 % | + 224.71 % | +| undici - dispatch | 10 | 6397.00 req/sec | ± 1.48 % | + 224.98 % | + ## Quick Start