From eb5f53129c81ee68165d0c097ca6391b636d58e7 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 3 Apr 2024 10:28:35 +0200 Subject: [PATCH 1/3] chore: migrate extract tests to use snapshot testing --- test/__snapshots__/extract.spec.ts.snap | 845 ++++++++++++++++++ .../{pytest_output.txt => pytest_output.json} | 0 test/extract.spec.ts | 587 +----------- 3 files changed, 879 insertions(+), 553 deletions(-) create mode 100644 test/__snapshots__/extract.spec.ts.snap rename test/data/extract/{pytest_output.txt => pytest_output.json} (100%) diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap new file mode 100644 index 000000000..cb0b4bf6f --- /dev/null +++ b/test/__snapshots__/extract.spec.ts.snap @@ -0,0 +1,845 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`extractResult() extracts benchmark output from benchmarkdotnet - benchmarkdotnet.json 1`] = ` +{ + "benches": [ + { + "name": "Sample.Benchmarks.Fib10", + "range": "± 0.1981212530783709", + "unit": "ns", + "value": 24.4202085009643, + }, + { + "name": "Sample.Benchmarks.Fib20", + "range": "± 0.7903737021529575", + "unit": "ns", + "value": 51.52008151549559, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "benchmarkdotnet", +} +`; + +exports[`extractResult() extracts benchmark output from benchmarkjs - benchmarkjs_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "93 samples", + "name": "fib(10)", + "range": "±0.74%", + "unit": "ops/sec", + "value": 1431759, + }, + { + "extra": "96 samples", + "name": "fib(20)", + "range": "±0.32%", + "unit": "ops/sec", + "value": 12146, + }, + { + "extra": "69 samples", + "name": "createObjectBuffer with 200 comments", + "range": "±1.70%", + "unit": "ops/sec", + "value": 81.61, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "benchmarkjs", +} +`; + +exports[`extractResult() extracts benchmark output from benchmarkluau - benchmarkluau_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "luau", + "name": "base64", + "range": "±0.636%", + "unit": "ms", + "value": 15.041, + }, + { + "extra": "luau", + "name": "chess", + "range": "±0.212%", + "unit": "ms", + "value": 69.56, + }, + { + "extra": "luau", + "name": "life", + "range": "±0.187%", + "unit": "ms", + "value": 85.089, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "benchmarkluau", +} +`; + +exports[`extractResult() extracts benchmark output from cargo - cargo_output.txt 1`] = ` +{ + "benches": [ + { + "name": "bench_fib_10", + "range": "± 24", + "unit": "ns/iter", + "value": 135, + }, + { + "name": "bench_fib_20", + "range": "± 755", + "unit": "ns/iter", + "value": 18149, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "cargo", +} +`; + +exports[`extractResult() extracts benchmark output from cargo - cargo_output2.txt 1`] = ` +{ + "benches": [ + { + "name": "bench_engine_new", + "range": "± 70126", + "unit": "ns/iter", + "value": 211834, + }, + { + "name": "bench_engine_new_raw", + "range": "± 18", + "unit": "ns/iter", + "value": 197, + }, + { + "name": "bench_engine_new_raw_core", + "range": "± 31", + "unit": "ns/iter", + "value": 196, + }, + { + "name": "bench_engine_register_fn", + "range": "± 82", + "unit": "ns/iter", + "value": 493, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "cargo", +} +`; + +exports[`extractResult() extracts benchmark output from cargo - criterion_output.txt 1`] = ` +{ + "benches": [ + { + "name": "Create Realm", + "range": "± 4", + "unit": "ns/iter", + "value": 329, + }, + { + "name": "Symbols (Execution)", + "range": "± 47", + "unit": "ns/iter", + "value": 3268, + }, + { + "name": "For loop (Execution)", + "range": "± 123", + "unit": "ns/iter", + "value": 12314, + }, + { + "name": "Fibonacci (Execution)", + "range": "± 10166", + "unit": "ns/iter", + "value": 1672496, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "cargo", +} +`; + +exports[`extractResult() extracts benchmark output from catch2 - catch2_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "100 samples +208 iterations", + "name": "Fibonacci 10", + "range": "± 19", + "unit": "ns", + "value": 344, + }, + { + "extra": "100 samples +2 iterations", + "name": "Fibonacci 20", + "range": "± 3.256", + "unit": "us", + "value": 41.731, + }, + { + "extra": "100 samples +1961 iterations", + "name": "Fibonacci~ 5!", + "range": "± 4", + "unit": "ns", + "value": 36, + }, + { + "extra": "100 samples +20 iterations", + "name": "Fibonacci-15_bench", + "range": "± 362", + "unit": "us", + "value": 3.789, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "catch2", +} +`; + +exports[`extractResult() extracts benchmark output from catch2 - issue16_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "100 samples +76353 iterations", + "name": "Fibonacci 10", + "range": "± 0", + "unit": "ns", + "value": 0, + }, + { + "extra": "100 samples +75814 iterations", + "name": "Fibonacci 20", + "range": "± 0", + "unit": "ns", + "value": 1, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "catch2", +} +`; + +exports[`extractResult() extracts benchmark output from customBiggerIsBetter - customBiggerIsBetter_output.txt 1`] = ` +{ + "benches": [ + { + "extra": undefined, + "name": "My Custom Bigger Is Better Benchmark - Throughput", + "range": undefined, + "unit": "req/s", + "value": 70, + }, + { + "extra": "Optional Value #1: 25 +Helpful Num #2: 100 +Anything Else!", + "name": "My Custom Bigger Is Better Benchmark - Free Memory", + "range": "3", + "unit": "Megabytes", + "value": 150, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "customBiggerIsBetter", +} +`; + +exports[`extractResult() extracts benchmark output from customSmallerIsBetter - customSmallerIsBetter_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "My Optional Information for the tooltip", + "name": "My Custom Smaller Is Better Benchmark - CPU Load", + "range": "5%", + "unit": "Percent", + "value": 50, + }, + { + "extra": undefined, + "name": "My Custom Smaller Is Better Benchmark - Memory Used", + "range": undefined, + "unit": "Megabytes", + "value": 100, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "customSmallerIsBetter", +} +`; + +exports[`extractResult() extracts benchmark output from go - go_fiber_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "4846809 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"}", + "unit": "ns/op 0 B/op 0 allocs/op", + "value": 247.6, + }, + { + "extra": "4846809 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"} - ns/op", + "unit": "ns/op", + "value": 247.6, + }, + { + "extra": "4846809 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"} - B/op", + "unit": "B/op", + "value": 0, + }, + { + "extra": "4846809 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"} - allocs/op", + "unit": "allocs/op", + "value": 0, + }, + { + "extra": "3900769 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"}", + "unit": "ns/op 0 B/op 0 allocs/op", + "value": 307.1, + }, + { + "extra": "3900769 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - ns/op", + "unit": "ns/op", + "value": 307.1, + }, + { + "extra": "3900769 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - B/op", + "unit": "B/op", + "value": 0, + }, + { + "extra": "3900769 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - allocs/op", + "unit": "allocs/op", + "value": 0, + }, + { + "extra": "5118646 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"}", + "unit": "ns/op 0 B/op 0 allocs/op", + "value": 316.1, + }, + { + "extra": "5118646 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - ns/op", + "unit": "ns/op", + "value": 316.1, + }, + { + "extra": "5118646 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - B/op", + "unit": "B/op", + "value": 0, + }, + { + "extra": "5118646 times +4 procs", + "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - allocs/op", + "unit": "allocs/op", + "value": 0, + }, + { + "extra": "3067818 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#01", + "unit": "ns/op 48 B/op 2 allocs/op", + "value": 390.7, + }, + { + "extra": "3067818 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#01 - ns/op", + "unit": "ns/op", + "value": 390.7, + }, + { + "extra": "3067818 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#01 - B/op", + "unit": "B/op", + "value": 48, + }, + { + "extra": "3067818 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#01 - allocs/op", + "unit": "allocs/op", + "value": 2, + }, + { + "extra": "1992943 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#02", + "unit": "ns/op 48 B/op 2 allocs/op", + "value": 602.1, + }, + { + "extra": "1992943 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#02 - ns/op", + "unit": "ns/op", + "value": 602.1, + }, + { + "extra": "1992943 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#02 - B/op", + "unit": "B/op", + "value": 48, + }, + { + "extra": "1992943 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#02 - allocs/op", + "unit": "allocs/op", + "value": 2, + }, + { + "extra": "4180965 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#03", + "unit": "ns/op 0 B/op 0 allocs/op", + "value": 286.3, + }, + { + "extra": "4180965 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#03 - ns/op", + "unit": "ns/op", + "value": 286.3, + }, + { + "extra": "4180965 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#03 - B/op", + "unit": "B/op", + "value": 0, + }, + { + "extra": "4180965 times +4 procs", + "name": "Benchmark_Utils_GetOffer/mime_extension#03 - allocs/op", + "unit": "allocs/op", + "value": 0, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "go", +} +`; + +exports[`extractResult() extracts benchmark output from go - go_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "5000000 times +8 procs", + "name": "BenchmarkFib10", + "unit": "ns/op", + "value": 325, + }, + { + "extra": "30000 times", + "name": "BenchmarkFib20", + "unit": "ns/op", + "value": 40537.123, + }, + { + "extra": "5000000 times +8 procs", + "name": "BenchmarkFib/my_tabled_benchmark_-_10", + "unit": "ns/op", + "value": 325, + }, + { + "extra": "30000 times", + "name": "BenchmarkFib/my_tabled_benchmark_-_20", + "unit": "ns/op", + "value": 40537.123, + }, + { + "extra": "30001 times", + "name": "BenchmarkFib/my/tabled/benchmark_-_20", + "unit": "ns/op", + "value": 40537.456, + }, + { + "extra": "30001 times", + "name": "BenchmarkFib/my/tabled/benchmark_-_20,var1=13,var2=14", + "unit": "ns/op", + "value": 40537.456, + }, + { + "extra": "4587789 times +16 procs", + "name": "BenchmarkFib11", + "unit": "ns/op 3.000 auxMetricUnits", + "value": 262.7, + }, + { + "extra": "4587789 times +16 procs", + "name": "BenchmarkFib11 - ns/op", + "unit": "ns/op", + "value": 262.7, + }, + { + "extra": "4587789 times +16 procs", + "name": "BenchmarkFib11 - auxMetricUnits", + "unit": "auxMetricUnits", + "value": 3, + }, + { + "extra": "37653 times +16 procs", + "name": "BenchmarkFib22", + "unit": "ns/op 4.000 auxMetricUnits", + "value": 31915, + }, + { + "extra": "37653 times +16 procs", + "name": "BenchmarkFib22 - ns/op", + "unit": "ns/op", + "value": 31915, + }, + { + "extra": "37653 times +16 procs", + "name": "BenchmarkFib22 - auxMetricUnits", + "unit": "auxMetricUnits", + "value": 4, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "go", +} +`; + +exports[`extractResult() extracts benchmark output from googlecpp - googlecpp_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "iterations: 3070566 +cpu: 213.65507206163295 ns +threads: 1", + "name": "fib_10", + "unit": "ns/iter", + "value": 214.98980114547953, + }, + { + "extra": "iterations: 23968 +cpu: 27364.90320427236 ns +threads: 1", + "name": "fib_20", + "unit": "ns/iter", + "value": 27455.600415007055, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "googlecpp", +} +`; + +exports[`extractResult() extracts benchmark output from jmh - jmh_output.json 1`] = ` +{ + "benches": [ + { + "extra": "iterations: 3 +forks: 1 +threads: 1", + "name": "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere", + "unit": "ops/s", + "value": 3376238873.1228185, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "jmh", +} +`; + +exports[`extractResult() extracts benchmark output from jmh - jmh_output_2.json 1`] = ` +{ + "benches": [ + { + "extra": "iterations: 3 +forks: 1 +threads: 1", + "name": "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere ( {"paramA":"17","paramB":"33"} )", + "unit": "ops/s", + "value": 3376238873.1228185, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "jmh", +} +`; + +exports[`extractResult() extracts benchmark output from julia - julia_output.json 1`] = ` +{ + "benches": [ + { + "extra": "gctime=0 +memory=0 +allocs=0 +params={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":390,"gcsample":false,"seconds":5,"overhead":0,"memory_tolerance":0.01}", + "name": "fib/10", + "unit": "ns", + "value": 246.03846153846155, + }, + { + "extra": "gctime=0 +memory=0 +allocs=0 +params={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5,"overhead":0,"memory_tolerance":0.01}", + "name": "fib/20", + "unit": "ns", + "value": 31028, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "julia", +} +`; + +exports[`extractResult() extracts benchmark output from pytest - pytest_output.json 1`] = ` +{ + "benches": [ + { + "extra": "mean: 24.08868133322941 usec +rounds: 38523", + "name": "bench.py::test_fib_10", + "range": "stddev: 0.000006175090189861328", + "unit": "iter/sec", + "value": 41513.272817492856, + }, + { + "extra": "mean: 2.9850306726618627 msec +rounds: 278", + "name": "bench.py::test_fib_20", + "range": "stddev: 0.0001745301654140968", + "unit": "iter/sec", + "value": 335.0049328331567, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "pytest", +} +`; + +exports[`extractResult() extracts benchmark output from pytest - pytest_several_units.json 1`] = ` +{ + "benches": [ + { + "extra": "mean: 149.95610248628836 nsec +rounds: 68536", + "name": "bench.py::test_fib_1", + "range": "stddev: 2.9351731952139377e-8", + "unit": "iter/sec", + "value": 6668618.238403659, + }, + { + "extra": "mean: 28.85754012484424 usec +rounds: 20025", + "name": "bench.py::test_fib_10", + "range": "stddev: 0.000005235937482008476", + "unit": "iter/sec", + "value": 34652.98828915334, + }, + { + "extra": "mean: 3.611916368852473 msec +rounds: 122", + "name": "bench.py::test_fib_20", + "range": "stddev: 0.0003737982822178215", + "unit": "iter/sec", + "value": 276.8613383807958, + }, + { + "extra": "mean: 2.0038430469999997 sec +rounds: 5", + "name": "bench.py::test_sleep_2", + "range": "stddev: 0.0018776587251587858", + "unit": "iter/sec", + "value": 0.49904108083570886, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "pytest", +} +`; diff --git a/test/data/extract/pytest_output.txt b/test/data/extract/pytest_output.json similarity index 100% rename from test/data/extract/pytest_output.txt rename to test/data/extract/pytest_output.json diff --git a/test/extract.spec.ts b/test/extract.spec.ts index f653f940f..f5fdb2dbf 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -1,7 +1,7 @@ import * as path from 'path'; import { strict as A } from 'assert'; import { Config, ToolType } from '../src/config'; -import { BenchmarkResult, extractResult } from '../src/extract'; +import { extractResult } from '../src/extract'; const dummyWebhookPayload = { head_commit: { @@ -57,604 +57,97 @@ describe('extractResult()', function () { const normalCases: Array<{ tool: ToolType; - expected: BenchmarkResult[]; - file?: string; + file: string; }> = [ { tool: 'cargo', - expected: [ - { - name: 'bench_fib_10', - range: '± 24', - unit: 'ns/iter', - value: 135, - }, - { - name: 'bench_fib_20', - range: '± 755', - unit: 'ns/iter', - value: 18149, - }, - ], + file: 'cargo_output.txt', }, { tool: 'cargo', file: 'cargo_output2.txt', - expected: [ - { - name: 'bench_engine_new', - range: '± 70126', - unit: 'ns/iter', - value: 211834, - }, - { - name: 'bench_engine_new_raw', - range: '± 18', - unit: 'ns/iter', - value: 197, - }, - { - name: 'bench_engine_new_raw_core', - range: '± 31', - unit: 'ns/iter', - value: 196, - }, - { - name: 'bench_engine_register_fn', - range: '± 82', - unit: 'ns/iter', - value: 493, - }, - ], }, { tool: 'cargo', file: 'criterion_output.txt', - expected: [ - { - name: 'Create Realm', - range: '± 4', - unit: 'ns/iter', - value: 329, - }, - { - name: 'Symbols (Execution)', - range: '± 47', - unit: 'ns/iter', - value: 3268, - }, - { - name: 'For loop (Execution)', - range: '± 123', - unit: 'ns/iter', - value: 12314, - }, - { - name: 'Fibonacci (Execution)', - range: '± 10166', - unit: 'ns/iter', - value: 1672496, - }, - ], }, { tool: 'catch2', - expected: [ - { - name: 'Fibonacci 10', - range: '± 19', - unit: 'ns', - value: 344, - extra: '100 samples\n208 iterations', - }, - { - name: 'Fibonacci 20', - range: '± 3.256', - unit: 'us', - value: 41.731, - extra: '100 samples\n2 iterations', - }, - { - name: 'Fibonacci~ 5!', - range: '± 4', - unit: 'ns', - value: 36, - extra: '100 samples\n1961 iterations', - }, - { - name: 'Fibonacci-15_bench', - range: '± 362', - unit: 'us', - value: 3.789, - extra: '100 samples\n20 iterations', - }, - ], + file: 'catch2_output.txt', }, { tool: 'go', - expected: [ - { - name: 'BenchmarkFib10', - unit: 'ns/op', - value: 325, - extra: '5000000 times\n8 procs', - }, - { - name: 'BenchmarkFib20', - unit: 'ns/op', - value: 40537.123, - extra: '30000 times', - }, - { - name: 'BenchmarkFib/my_tabled_benchmark_-_10', - unit: 'ns/op', - value: 325, - extra: '5000000 times\n8 procs', - }, - { - name: 'BenchmarkFib/my_tabled_benchmark_-_20', - unit: 'ns/op', - value: 40537.123, - extra: '30000 times', - }, - { - name: 'BenchmarkFib/my/tabled/benchmark_-_20', - unit: 'ns/op', - value: 40537.456, - extra: '30001 times', - }, - { - name: 'BenchmarkFib/my/tabled/benchmark_-_20,var1=13,var2=14', - unit: 'ns/op', - value: 40537.456, - extra: '30001 times', - }, - { - name: 'BenchmarkFib11', - unit: 'ns/op\t 3.000 auxMetricUnits', - value: 262.7, - extra: '4587789 times\n16 procs', - }, - { - name: 'BenchmarkFib11 - ns/op', - unit: 'ns/op', - value: 262.7, - extra: '4587789 times\n16 procs', - }, - { - name: 'BenchmarkFib11 - auxMetricUnits', - unit: 'auxMetricUnits', - value: 3, - extra: '4587789 times\n16 procs', - }, - { - name: 'BenchmarkFib22', - unit: 'ns/op\t 4.000 auxMetricUnits', - value: 31915, - extra: '37653 times\n16 procs', - }, - { - name: 'BenchmarkFib22 - ns/op', - unit: 'ns/op', - value: 31915, - extra: '37653 times\n16 procs', - }, - { - name: 'BenchmarkFib22 - auxMetricUnits', - unit: 'auxMetricUnits', - value: 4, - extra: '37653 times\n16 procs', - }, - ], + file: 'go_output.txt', }, { tool: 'go', file: 'go_fiber_output.txt', - expected: [ - { - name: `Benchmark_Ctx_Accepts/run-[]string{".xml"}`, - unit: 'ns/op 0 B/op 0 allocs/op', - value: 247.6, - extra: '4846809 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{".xml"} - ns/op`, - unit: 'ns/op', - value: 247.6, - extra: '4846809 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{".xml"} - B/op`, - unit: 'B/op', - value: 0, - extra: '4846809 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{".xml"} - allocs/op`, - unit: 'allocs/op', - value: 0, - extra: '4846809 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"}`, - unit: 'ns/op 0 B/op 0 allocs/op', - value: 307.1, - extra: '3900769 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - ns/op`, - unit: 'ns/op', - value: 307.1, - extra: '3900769 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - B/op`, - unit: 'B/op', - value: 0, - extra: '3900769 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - allocs/op`, - unit: 'allocs/op', - value: 0, - extra: '3900769 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"}`, - unit: 'ns/op 0 B/op 0 allocs/op', - value: 316.1, - extra: '5118646 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - ns/op`, - unit: 'ns/op', - value: 316.1, - extra: '5118646 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - B/op`, - unit: 'B/op', - value: 0, - extra: '5118646 times\n4 procs', - }, - { - name: `Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - allocs/op`, - unit: 'allocs/op', - value: 0, - extra: '5118646 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#01`, - unit: 'ns/op 48 B/op 2 allocs/op', - value: 390.7, - extra: '3067818 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#01 - ns/op`, - unit: 'ns/op', - value: 390.7, - extra: '3067818 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#01 - B/op`, - unit: 'B/op', - value: 48, - extra: '3067818 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#01 - allocs/op`, - unit: 'allocs/op', - value: 2, - extra: '3067818 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#02`, - unit: 'ns/op 48 B/op 2 allocs/op', - value: 602.1, - extra: '1992943 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#02 - ns/op`, - unit: 'ns/op', - value: 602.1, - extra: '1992943 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#02 - B/op`, - unit: 'B/op', - value: 48, - extra: '1992943 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#02 - allocs/op`, - unit: 'allocs/op', - value: 2, - extra: '1992943 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#03`, - unit: 'ns/op 0 B/op 0 allocs/op', - value: 286.3, - extra: '4180965 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#03 - ns/op`, - unit: 'ns/op', - value: 286.3, - extra: '4180965 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#03 - B/op`, - unit: 'B/op', - value: 0, - extra: '4180965 times\n4 procs', - }, - { - name: `Benchmark_Utils_GetOffer/mime_extension#03 - allocs/op`, - unit: 'allocs/op', - value: 0, - extra: '4180965 times\n4 procs', - }, - ], }, { tool: 'benchmarkjs', - expected: [ - { - name: 'fib(10)', - range: '±0.74%', - unit: 'ops/sec', - value: 1431759, - extra: '93 samples', - }, - { - name: 'fib(20)', - range: '±0.32%', - unit: 'ops/sec', - value: 12146, - extra: '96 samples', - }, - { - name: 'createObjectBuffer with 200 comments', - range: '±1.70%', - unit: 'ops/sec', - value: 81.61, - extra: '69 samples', - }, - ], + file: 'benchmarkjs_output.txt', }, { tool: 'benchmarkluau', - expected: [ - { - name: 'base64', - range: '±0.636%', - unit: 'ms', - value: 15.041, - extra: 'luau', - }, - { - name: 'chess', - range: '±0.212%', - unit: 'ms', - value: 69.56, - extra: 'luau', - }, - { - name: 'life', - range: '±0.187%', - unit: 'ms', - value: 85.089, - extra: 'luau', - }, - ], + file: 'benchmarkluau_output.txt', }, { tool: 'pytest', - expected: [ - { - name: 'bench.py::test_fib_10', - range: 'stddev: 0.000006175090189861328', - unit: 'iter/sec', - value: 41513.272817492856, - extra: 'mean: 24.08868133322941 usec\nrounds: 38523', - }, - { - name: 'bench.py::test_fib_20', - range: 'stddev: 0.0001745301654140968', - unit: 'iter/sec', - value: 335.0049328331567, - extra: 'mean: 2.9850306726618627 msec\nrounds: 278', - }, - ], + file: 'pytest_output.json', }, { tool: 'googlecpp', - expected: [ - { - extra: 'iterations: 3070566\ncpu: 213.65507206163295 ns\nthreads: 1', - name: 'fib_10', - unit: 'ns/iter', - value: 214.98980114547953, - }, - { - extra: 'iterations: 23968\ncpu: 27364.90320427236 ns\nthreads: 1', - name: 'fib_20', - unit: 'ns/iter', - value: 27455.600415007055, - }, - ], + file: 'googlecpp_output.txt', }, { tool: 'pytest', file: 'pytest_several_units.json', - expected: [ - { - extra: 'mean: 149.95610248628836 nsec\nrounds: 68536', - name: 'bench.py::test_fib_1', - range: 'stddev: 2.9351731952139377e-8', - unit: 'iter/sec', - value: 6668618.238403659, - }, - { - name: 'bench.py::test_fib_10', - range: 'stddev: 0.000005235937482008476', - unit: 'iter/sec', - value: 34652.98828915334, - extra: 'mean: 28.85754012484424 usec\nrounds: 20025', - }, - { - name: 'bench.py::test_fib_20', - range: 'stddev: 0.0003737982822178215', - unit: 'iter/sec', - value: 276.8613383807958, - extra: 'mean: 3.611916368852473 msec\nrounds: 122', - }, - { - extra: 'mean: 2.0038430469999997 sec\nrounds: 5', - name: 'bench.py::test_sleep_2', - range: 'stddev: 0.0018776587251587858', - unit: 'iter/sec', - value: 0.49904108083570886, - }, - ], }, { tool: 'catch2', file: 'issue16_output.txt', - expected: [ - { - extra: '100 samples\n76353 iterations', - name: 'Fibonacci 10', - range: '± 0', - unit: 'ns', - value: 0, - }, - { - extra: '100 samples\n75814 iterations', - name: 'Fibonacci 20', - range: '± 0', - unit: 'ns', - value: 1, - }, - ], }, { tool: 'julia', file: 'julia_output.json', - expected: [ - { - extra: 'gctime=0\nmemory=0\nallocs=0\nparams={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":390,"gcsample":false,"seconds":5,"overhead":0,"memory_tolerance":0.01}', - name: 'fib/10', - unit: 'ns', - value: 246.03846153846155, - }, - { - extra: 'gctime=0\nmemory=0\nallocs=0\nparams={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5,"overhead":0,"memory_tolerance":0.01}', - name: 'fib/20', - unit: 'ns', - value: 31028, - }, - ], }, { tool: 'jmh', file: 'jmh_output.json', - expected: [ - { - extra: 'iterations: 3\nforks: 1\nthreads: 1', - name: 'org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere', - unit: 'ops/s', - value: 3.3762388731228185e9, - }, - ], }, { tool: 'jmh', file: 'jmh_output_2.json', - expected: [ - { - extra: 'iterations: 3\nforks: 1\nthreads: 1', - name: 'org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere ( {"paramA":"17","paramB":"33"} )', - unit: 'ops/s', - value: 3.3762388731228185e9, - }, - ], }, { tool: 'benchmarkdotnet', file: 'benchmarkdotnet.json', - expected: [ - { - name: 'Sample.Benchmarks.Fib10', - range: '± 0.1981212530783709', - unit: 'ns', - value: 24.4202085009643, - }, - { - name: 'Sample.Benchmarks.Fib20', - range: '± 0.7903737021529575', - unit: 'ns', - value: 51.52008151549559, - }, - ], }, { tool: 'customBiggerIsBetter', - expected: [ - { - name: 'My Custom Bigger Is Better Benchmark - Throughput', - unit: 'req/s', - value: 70, - range: undefined, - extra: undefined, - }, - { - name: 'My Custom Bigger Is Better Benchmark - Free Memory', - unit: 'Megabytes', - value: 150, - range: '3', - extra: 'Optional Value #1: 25\nHelpful Num #2: 100\nAnything Else!', - }, - ], + file: 'customBiggerIsBetter_output.txt', }, { tool: 'customSmallerIsBetter', - expected: [ - { - name: 'My Custom Smaller Is Better Benchmark - CPU Load', - unit: 'Percent', - value: 50, - range: '5%', - extra: 'My Optional Information for the tooltip', - }, - { - name: 'My Custom Smaller Is Better Benchmark - Memory Used', - unit: 'Megabytes', - value: 100, - range: undefined, - extra: undefined, - }, - ], + file: 'customSmallerIsBetter_output.txt', }, ]; - for (const test of normalCases) { - it(`extracts benchmark output from ${test.tool}${test.file ? ` - ${test.file}` : ''}`, async function () { - const file = test.file ?? `${test.tool}_output.txt`; - const outputFilePath = path.join(__dirname, 'data', 'extract', file); - const config = { - tool: test.tool, - outputFilePath, - } as Config; - const bench = await extractResult(config); - - A.equal(bench.commit, dummyWebhookPayload.head_commit); - A.ok(bench.date <= Date.now(), bench.date.toString()); - A.equal(bench.tool, test.tool); - A.deepEqual(bench.benches, test.expected); + it.each(normalCases)(`extracts benchmark output from $tool - $file`, async function (test) { + jest.useFakeTimers({ + now: 1712131503296, }); - } + const outputFilePath = path.join(__dirname, 'data', 'extract', test.file); + const config = { + tool: test.tool, + outputFilePath, + } as Config; + const bench = await extractResult(config); + + expect(bench).toMatchSnapshot(); + + jest.useRealTimers(); + }); it('raises an error on unexpected tool', async function () { const config = { @@ -680,28 +173,16 @@ describe('extractResult()', function () { await A.rejects(extractResult(config), /^Error: No benchmark result was found in /); }); - const toolSpecificErrorCases: Array<{ - it: string; - tool: ToolType; - file: string; - expected: RegExp; - }> = [ - ...(['pytest', 'googlecpp', 'jmh', 'customBiggerIsBetter', 'customSmallerIsBetter'] as const).map((tool) => ({ - it: `raises an error when output file is not in JSON with tool '${tool}'`, - tool, - file: 'go_output.txt', - expected: /must be JSON file/, - })), - ]; - - for (const t of toolSpecificErrorCases) { - it(t.it, async function () { + it.each(['pytest', 'googlecpp', 'jmh', 'customBiggerIsBetter', 'customSmallerIsBetter'] as const)( + `raises an error when output file is not in JSON with tool '%s'`, + async function (tool) { + const file = 'go_output.txt'; // Note: go_output.txt is not in JSON format! - const outputFilePath = path.join(__dirname, 'data', 'extract', t.file); - const config = { tool: t.tool, outputFilePath } as Config; - await A.rejects(extractResult(config), t.expected); - }); - } + const outputFilePath = path.join(__dirname, 'data', 'extract', file); + const config = { tool, outputFilePath } as Config; + await A.rejects(extractResult(config), /must be JSON file/); + }, + ); it('collects the commit information from pull_request payload as fallback', async function () { dummyGitHubContext.payload = { From 9591acd428cf6219e1ab50050a0e4aee574f92f3 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 3 Apr 2024 10:31:41 +0200 Subject: [PATCH 2/3] use proper json files --- test/__snapshots__/extract.spec.ts.snap | 6 +++--- ...IsBetter_output.txt => customBiggerIsBetter_output.json} | 0 ...sBetter_output.txt => customSmallerIsBetter_output.json} | 0 .../extract/{googlecpp_output.txt => googlecpp_output.json} | 0 test/extract.spec.ts | 6 +++--- 5 files changed, 6 insertions(+), 6 deletions(-) rename test/data/extract/{customBiggerIsBetter_output.txt => customBiggerIsBetter_output.json} (100%) rename test/data/extract/{customSmallerIsBetter_output.txt => customSmallerIsBetter_output.json} (100%) rename test/data/extract/{googlecpp_output.txt => googlecpp_output.json} (100%) diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index cb0b4bf6f..d37f347a7 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -298,7 +298,7 @@ exports[`extractResult() extracts benchmark output from catch2 - issue16_output. } `; -exports[`extractResult() extracts benchmark output from customBiggerIsBetter - customBiggerIsBetter_output.txt 1`] = ` +exports[`extractResult() extracts benchmark output from customBiggerIsBetter - customBiggerIsBetter_output.json 1`] = ` { "benches": [ { @@ -331,7 +331,7 @@ Anything Else!", } `; -exports[`extractResult() extracts benchmark output from customSmallerIsBetter - customSmallerIsBetter_output.txt 1`] = ` +exports[`extractResult() extracts benchmark output from customSmallerIsBetter - customSmallerIsBetter_output.json 1`] = ` { "benches": [ { @@ -644,7 +644,7 @@ exports[`extractResult() extracts benchmark output from go - go_output.txt 1`] = } `; -exports[`extractResult() extracts benchmark output from googlecpp - googlecpp_output.txt 1`] = ` +exports[`extractResult() extracts benchmark output from googlecpp - googlecpp_output.json 1`] = ` { "benches": [ { diff --git a/test/data/extract/customBiggerIsBetter_output.txt b/test/data/extract/customBiggerIsBetter_output.json similarity index 100% rename from test/data/extract/customBiggerIsBetter_output.txt rename to test/data/extract/customBiggerIsBetter_output.json diff --git a/test/data/extract/customSmallerIsBetter_output.txt b/test/data/extract/customSmallerIsBetter_output.json similarity index 100% rename from test/data/extract/customSmallerIsBetter_output.txt rename to test/data/extract/customSmallerIsBetter_output.json diff --git a/test/data/extract/googlecpp_output.txt b/test/data/extract/googlecpp_output.json similarity index 100% rename from test/data/extract/googlecpp_output.txt rename to test/data/extract/googlecpp_output.json diff --git a/test/extract.spec.ts b/test/extract.spec.ts index f5fdb2dbf..2e8817e2b 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -97,7 +97,7 @@ describe('extractResult()', function () { }, { tool: 'googlecpp', - file: 'googlecpp_output.txt', + file: 'googlecpp_output.json', }, { tool: 'pytest', @@ -125,11 +125,11 @@ describe('extractResult()', function () { }, { tool: 'customBiggerIsBetter', - file: 'customBiggerIsBetter_output.txt', + file: 'customBiggerIsBetter_output.json', }, { tool: 'customSmallerIsBetter', - file: 'customSmallerIsBetter_output.txt', + file: 'customSmallerIsBetter_output.json', }, ]; From 02479e962f54ee02293a715a13ee142bc457e390 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 3 Apr 2024 10:51:34 +0200 Subject: [PATCH 3/3] use it.each where possible --- test/config.spec.ts | 114 ++++++++++----------- test/write.spec.ts | 239 +++++++++++++++++++++----------------------- 2 files changed, 172 insertions(+), 181 deletions(-) diff --git a/test/config.spec.ts b/test/config.spec.ts index 058139dcd..24094fd2e 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -46,7 +46,11 @@ describe('configFromJobInput()', function () { 'max-items-in-chart': '', }; - const validation_tests = [ + const validationTests: Array<{ + what: string; + inputs: Inputs; + expected: RegExp; + }> = [ { what: 'wrong name', inputs: { ...defaultInputs, name: '' }, @@ -158,18 +162,12 @@ describe('configFromJobInput()', function () { inputs: { ...defaultInputs, 'alert-threshold': '150%', 'fail-threshold': '120%' }, expected: /'alert-threshold' value must be smaller than 'fail-threshold' value but got 1.5 > 1.2/, }, - ] as Array<{ - what: string; - inputs: Inputs; - expected: RegExp; - }>; + ]; - for (const test of validation_tests) { - it(`validates ${test.what}`, async function () { - mockInputs(test.inputs); - await A.rejects(configFromJobInput, test.expected); - }); - } + it.each(validationTests)('validates $what', async function (test) { + mockInputs(test.inputs); + await A.rejects(configFromJobInput, test.expected); + }); interface ExpectedResult { name: string; @@ -205,7 +203,11 @@ describe('configFromJobInput()', function () { failThreshold: null, }; - const returnedConfigTests = [ + const returnedConfigTests: Array<{ + what: string; + inputs: any; + expected: ExpectedResult; + }> = [ ...VALID_TOOLS.map((tool: string) => ({ what: 'valid tool ' + tool, inputs: { ...defaultInputs, tool }, @@ -237,20 +239,24 @@ describe('configFromJobInput()', function () { inputs: { ...defaultInputs, 'gh-pages-branch': 'master' }, expected: { ...defaultExpected, ghPagesBranch: 'master' }, }, - ...[ - ['150%', 1.5], - ['0%', 0], - ['123.4%', 1.234], - ].map(([v, e]) => ({ + ...( + [ + ['150%', 1.5], + ['0%', 0], + ['123.4%', 1.234], + ] as Array<[string, number]> + ).map(([v, e]) => ({ what: `with alert threshold ${v}`, inputs: { ...defaultInputs, 'alert-threshold': v }, expected: { ...defaultExpected, alertThreshold: e }, })), - ...[ - ['@foo', ['@foo']], - ['@foo,@bar', ['@foo', '@bar']], - ['@foo, @bar ', ['@foo', '@bar']], - ].map(([v, e]) => ({ + ...( + [ + ['@foo', ['@foo']], + ['@foo,@bar', ['@foo', '@bar']], + ['@foo, @bar ', ['@foo', '@bar']], + ] as Array<[string, string[]]> + ).map(([v, e]) => ({ what: `with comment CC users ${v}`, inputs: { ...defaultInputs, 'alert-comment-cc-users': v }, expected: { ...defaultExpected, alertCommentCcUsers: e }, @@ -280,42 +286,36 @@ describe('configFromJobInput()', function () { }, expected: defaultExpected, }, - ] as Array<{ - what: string; - inputs: Inputs; - expected: ExpectedResult; - }>; + ]; - for (const test of returnedConfigTests) { - it(`returns validated config with ${test.what}`, async function () { - mockInputs(test.inputs); - const actual = await configFromJobInput(); - A.equal(actual.name, test.expected.name); - A.equal(actual.tool, test.expected.tool); - A.equal(actual.ghPagesBranch, test.expected.ghPagesBranch); - A.equal(actual.githubToken, test.expected.githubToken); - A.equal(actual.skipFetchGhPages, test.expected.skipFetchGhPages); - A.equal(actual.commentOnAlert, test.expected.commentOnAlert); - A.equal(actual.failOnAlert, test.expected.failOnAlert); - A.equal(actual.alertThreshold, test.expected.alertThreshold); - A.deepEqual(actual.alertCommentCcUsers, test.expected.alertCommentCcUsers); - A.ok(path.isAbsolute(actual.outputFilePath), actual.outputFilePath); - A.ok(path.isAbsolute(actual.benchmarkDataDirPath), actual.benchmarkDataDirPath); - A.equal(actual.maxItemsInChart, test.expected.maxItemsInChart); - if (test.expected.failThreshold === null) { - A.equal(actual.failThreshold, test.expected.alertThreshold); - } else { - A.equal(actual.failThreshold, test.expected.failThreshold); - } + it.each(returnedConfigTests)('returns validated config with $what', async function (test) { + mockInputs(test.inputs); + const actual = await configFromJobInput(); + A.equal(actual.name, test.expected.name); + A.equal(actual.tool, test.expected.tool); + A.equal(actual.ghPagesBranch, test.expected.ghPagesBranch); + A.equal(actual.githubToken, test.expected.githubToken); + A.equal(actual.skipFetchGhPages, test.expected.skipFetchGhPages); + A.equal(actual.commentOnAlert, test.expected.commentOnAlert); + A.equal(actual.failOnAlert, test.expected.failOnAlert); + A.equal(actual.alertThreshold, test.expected.alertThreshold); + A.deepEqual(actual.alertCommentCcUsers, test.expected.alertCommentCcUsers); + A.ok(path.isAbsolute(actual.outputFilePath), actual.outputFilePath); + A.ok(path.isAbsolute(actual.benchmarkDataDirPath), actual.benchmarkDataDirPath); + A.equal(actual.maxItemsInChart, test.expected.maxItemsInChart); + if (test.expected.failThreshold === null) { + A.equal(actual.failThreshold, test.expected.alertThreshold); + } else { + A.equal(actual.failThreshold, test.expected.failThreshold); + } - if (test.expected.hasExternalDataJsonPath) { - A.equal(typeof actual.externalDataJsonPath, 'string'); - A.ok(path.isAbsolute(actual.externalDataJsonPath as string), actual.externalDataJsonPath); - } else { - A.equal(actual.externalDataJsonPath, undefined); - } - }); - } + if (test.expected.hasExternalDataJsonPath) { + A.equal(typeof actual.externalDataJsonPath, 'string'); + A.ok(path.isAbsolute(actual.externalDataJsonPath as string), actual.externalDataJsonPath); + } else { + A.equal(actual.externalDataJsonPath, undefined); + } + }); it('resolves relative paths in config', async function () { mockInputs({ diff --git a/test/write.spec.ts b/test/write.spec.ts index 121ce517b..e88543cbb 100644 --- a/test/write.spec.ts +++ b/test/write.spec.ts @@ -708,115 +708,108 @@ describe.each(['https://github.com', 'https://github.enterprise.corp'])('writeBe }, ]; - for (const t of normalCases) { - it(t.it, async function () { - gitHubContext.payload.repository = { - private: false, - html_url: `${serverUrl}/user/repo`, - } as RepositoryPayloadSubset | null; + it.each(normalCases)('$it', async function (t) { + gitHubContext.payload.repository = { + private: false, + html_url: `${serverUrl}/user/repo`, + } as RepositoryPayloadSubset | null; - if (t.repoPayload !== undefined) { - gitHubContext.payload.repository = t.repoPayload; - } - if (t.data !== null) { - await fs.writeFile(dataJson, JSON.stringify(t.data), 'utf8'); - } + if (t.repoPayload !== undefined) { + gitHubContext.payload.repository = t.repoPayload; + } + if (t.data !== null) { + await fs.writeFile(dataJson, JSON.stringify(t.data), 'utf8'); + } - let caughtError: Error | null = null; - try { - await writeBenchmark(t.added, t.config); - } catch (err: any) { - if (!t.error && !t.commitComment) { - throw err; - } - caughtError = err; + let caughtError: Error | null = null; + try { + await writeBenchmark(t.added, t.config); + } catch (err: any) { + if (!t.error && !t.commitComment) { + throw err; } + caughtError = err; + } - const json: DataJson = JSON.parse(await fs.readFile(dataJson, 'utf8')); + const json: DataJson = JSON.parse(await fs.readFile(dataJson, 'utf8')); - expect('number').toEqual(typeof json.lastUpdate); - expect(json.entries[t.config.name]).toBeTruthy(); - const len = json.entries[t.config.name].length; - ok(len > 0); - expect(t.added).toEqual(json.entries[t.config.name][len - 1]); // Check last item is the newest + expect('number').toEqual(typeof json.lastUpdate); + expect(json.entries[t.config.name]).toBeTruthy(); + const len = json.entries[t.config.name].length; + ok(len > 0); + expect(t.added).toEqual(json.entries[t.config.name][len - 1]); // Check last item is the newest - if (t.data !== null) { - ok(json.lastUpdate > t.data.lastUpdate); - expect(t.data.repoUrl).toEqual(json.repoUrl); - for (const name of Object.keys(t.data.entries)) { - const entries = t.data.entries[name]; - if (name === t.config.name) { - if (t.config.maxItemsInChart === null || len < t.config.maxItemsInChart) { - expect(entries.length + 1).toEqual(len); - // Check benchmark data except for the last appended one are not modified - expect(entries).toEqual(json.entries[name].slice(0, -1)); - } else { - // When data items was truncated due to max-items-in-chart - expect(entries.length).toEqual(len); // Number of items did not change because first item was shifted - expect(entries.slice(1)).toEqual(json.entries[name].slice(0, -1)); - } + if (t.data !== null) { + ok(json.lastUpdate > t.data.lastUpdate); + expect(t.data.repoUrl).toEqual(json.repoUrl); + for (const name of Object.keys(t.data.entries)) { + const entries = t.data.entries[name]; + if (name === t.config.name) { + if (t.config.maxItemsInChart === null || len < t.config.maxItemsInChart) { + expect(entries.length + 1).toEqual(len); + // Check benchmark data except for the last appended one are not modified + expect(entries).toEqual(json.entries[name].slice(0, -1)); } else { - expect(entries).toEqual(json.entries[name]); // eq(json.entries[name], entries, name); + // When data items was truncated due to max-items-in-chart + expect(entries.length).toEqual(len); // Number of items did not change because first item was shifted + expect(entries.slice(1)).toEqual(json.entries[name].slice(0, -1)); } + } else { + expect(entries).toEqual(json.entries[name]); // eq(json.entries[name], entries, name); } } + } - if (t.error) { - ok(caughtError); - const expected = t.error.join('\n'); - expect(caughtError.message).toEqual(expected); - } + if (t.error) { + ok(caughtError); + const expected = t.error.join('\n'); + expect(caughtError.message).toEqual(expected); + } - if (t.commitComment !== undefined) { - ok(caughtError); - // Last line is appended only for failure message - const messageLines = caughtError.message.split('\n'); - ok(messageLines.length > 0); - const expectedMessage = wrapBodyWithBenchmarkTags( - 'Test benchmark Alert', - messageLines.slice(0, -1).join('\n'), - ); - ok( - fakedRepos.spyOpts.length > 0, - `len: ${fakedRepos.spyOpts.length}, caught: ${caughtError.message}`, - ); - const opts = fakedRepos.lastCall(); - expect('user').toEqual(opts.owner); - expect('repo').toEqual(opts.repo); - expect('current commit id').toEqual(opts.commit_sha); - expect(expectedMessage).toEqual(opts.body); - const commentLine = messageLines[messageLines.length - 1]; - expect(t.commitComment).toEqual(commentLine); + if (t.commitComment !== undefined) { + ok(caughtError); + // Last line is appended only for failure message + const messageLines = caughtError.message.split('\n'); + ok(messageLines.length > 0); + const expectedMessage = wrapBodyWithBenchmarkTags( + 'Test benchmark Alert', + messageLines.slice(0, -1).join('\n'), + ); + ok(fakedRepos.spyOpts.length > 0, `len: ${fakedRepos.spyOpts.length}, caught: ${caughtError.message}`); + const opts = fakedRepos.lastCall(); + expect('user').toEqual(opts.owner); + expect('repo').toEqual(opts.repo); + expect('current commit id').toEqual(opts.commit_sha); + expect(expectedMessage).toEqual(opts.body); + const commentLine = messageLines[messageLines.length - 1]; + expect(t.commitComment).toEqual(commentLine); - // Check the body is a correct markdown document by markdown parser - // Validate markdown content via HTML - // TODO: Use Markdown AST instead of DOM API - const html = md2html.render(opts.body); - const query = cheerio.load(html); + // Check the body is a correct markdown document by markdown parser + // Validate markdown content via HTML + // TODO: Use Markdown AST instead of DOM API + const html = md2html.render(opts.body); + const query = cheerio.load(html); - const h1 = query('h1'); - expect(1).toEqual(h1.length); - expect(':warning: Performance Alert :warning:').toEqual(h1.text()); + const h1 = query('h1'); + expect(1).toEqual(h1.length); + expect(':warning: Performance Alert :warning:').toEqual(h1.text()); - const tr = query('tbody tr'); - expect(t.added.benches.length).toEqual(tr.length); + const tr = query('tbody tr'); + expect(t.added.benches.length).toEqual(tr.length); - const a = query('a'); - expect(2).toEqual(a.length); + const a = query('a'); + expect(2).toEqual(a.length); - const workflowLink = a.first(); - expect('workflow').toEqual(workflowLink.text()); - const workflowUrl = workflowLink.attr('href'); - ok(workflowUrl?.startsWith(json.repoUrl), workflowUrl); + const workflowLink = a.first(); + expect('workflow').toEqual(workflowLink.text()); + const workflowUrl = workflowLink.attr('href'); + ok(workflowUrl?.startsWith(json.repoUrl), workflowUrl); - const actionLink = a.last(); - expect('github-action-benchmark').toEqual(actionLink.text()); - expect('https://github.com/marketplace/actions/continuous-benchmark').toEqual( - actionLink.attr('href'), - ); - } - }); - } + const actionLink = a.last(); + expect('github-action-benchmark').toEqual(actionLink.text()); + expect('https://github.com/marketplace/actions/continuous-benchmark').toEqual(actionLink.attr('href')); + } + }); }); // Tests for updating GitHub Pages branch @@ -1311,44 +1304,42 @@ describe.each(['https://github.com', 'https://github.enterprise.corp'])('writeBe }, ]; - for (const t of retryCases) { - it(t.it, async function () { - gitSpy.pushFailure = t.pushErrorMessage; - gitSpy.pushFailureCount = t.pushErrorCount; - const config = { ...defaultCfg, benchmarkDataDirPath: 'with-index-html' }; - const added: Benchmark = { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 110)], - }; + it.each(retryCases)('$it', async function (t) { + gitSpy.pushFailure = t.pushErrorMessage; + gitSpy.pushFailureCount = t.pushErrorCount; + const config = { ...defaultCfg, benchmarkDataDirPath: 'with-index-html' }; + const added: Benchmark = { + commit: commit('current commit id'), + date: lastUpdate, + tool: 'cargo', + benches: [bench('bench_fib_10', 110)], + }; - const originalDataJs = path.join(config.benchmarkDataDirPath, 'original_data.js'); - const dataJs = path.join(config.benchmarkDataDirPath, 'data.js'); - await fs.copyFile(originalDataJs, dataJs); + const originalDataJs = path.join(config.benchmarkDataDirPath, 'original_data.js'); + const dataJs = path.join(config.benchmarkDataDirPath, 'data.js'); + await fs.copyFile(originalDataJs, dataJs); - const history = gitHistory({ dir: 'with-index-html', addIndexHtml: false }); - if (t.pushErrorCount > 0) { - // First 2 commands are fetch and switch. They are not repeated on retry - const retryHistory = history.slice(2, -1); - retryHistory.push(['cmd', [[], 'reset', '--hard', 'HEAD~1']]); + const history = gitHistory({ dir: 'with-index-html', addIndexHtml: false }); + if (t.pushErrorCount > 0) { + // First 2 commands are fetch and switch. They are not repeated on retry + const retryHistory = history.slice(2, -1); + retryHistory.push(['cmd', [[], 'reset', '--hard', 'HEAD~1']]); - const retries = Math.min(t.pushErrorCount, maxRetries); - for (let i = 0; i < retries; i++) { - history.splice(2, 0, ...retryHistory); - } + const retries = Math.min(t.pushErrorCount, maxRetries); + for (let i = 0; i < retries; i++) { + history.splice(2, 0, ...retryHistory); } + } - try { - await writeBenchmark(added, config); - expect(gitSpy.history).toEqual(history); - } catch (err: any) { - if (t.error === undefined) { - throw err; - } - ok(t.error.test(err.message), `'${err.message}' did not match to ${t.error}`); + try { + await writeBenchmark(added, config); + expect(gitSpy.history).toEqual(history); + } catch (err: any) { + if (t.error === undefined) { + throw err; } - }); - } + ok(t.error.test(err.message), `'${err.message}' did not match to ${t.error}`); + } + }); }); });