Add support for catch2 benchmarks

.github/workflows/catch2.yml

@@ -0,0 +1,29 @@
+name: Catch2 C++ Example
+on: push
+
+jobs:
+  benchmark:
+    name: Run C++ benchmark example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          cd examples/catch2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench > ../benchmark_result.txt
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Catch2 Benchmark
+          tool: "catch2"
+          output-file-path: examples/catch2/benchmark_result.txt
+          # Use personal access token instead of GITHUB_TOKEN due to https://github.hscsec.cnmunity/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false
+          github-token: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: "200%"
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: "@bernedom"

.github/workflows/ci.yml

@@ -160,6 +160,41 @@ jobs:
           skip-fetch-gh-pages: true
           fail-on-alert: true
       - run: node ./scripts/ci_validate_modification.js before_data.js 'C++ Benchmark'
+  catch2-framework:
+    name: Run Catch2 C++ Benchmark Framework example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+      - run: npm ci
+      - run: npm run build
+      - name: Save previous data.js
+        run: |
+          git fetch origin gh-pages
+          git checkout gh-pages
+          cp ./dev/bench/data.js before_data.js
+          git checkout -
+      - name: Run benchmark
+        run: |
+          cd examples/catch2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench > ../benchmark_result.txt
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Catch2 Benchmark
+          tool: "catch2"
+          output-file-path: examples/catch2/benchmark_result.txt
+          skip-fetch-gh-pages: true
+          fail-on-alert: true
+      - run: node ./scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
+
   only-alert-with-cache:
     name: Run alert check with actions/cache
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -3,15 +3,15 @@ Development
 
 ## How to add new benchmark tool support
 
-Adding support for new benchmaking tools is welcome!
+Adding support for new benchmarking tools is welcome!
 
 1. Add your tool name in `src/config.ts`
 2. Implement the logic to extract benchmark results from output in `src/extract.ts`
 3. Add tests for your tool under `test/*.ts`
 4. Add your tool's color in `default_index_html.ts`
 5. Add example project under `examples/` directory
 6. Add workflow to run the example project under `.github/workflows/` directory
-7. Update `.github/workflows/ci.yml` to check your tool works without an error
+7. Update `.github/workflows/ci.yml` to check your tool works without an error (see below for needed changes)
 8. Add README.md in the example project directory and update README.md at root directory
 
 Important part is 2.
@@ -29,6 +29,22 @@ And for another example, here are commits to add support for `pytest-benchmark`:
 - Add workflows for test and example: https://github.com/rhysd/github-action-benchmark/commit/1e4ebf2e9ecde9e7620661c60455b22837a2bdaf
 - Add documentation: https://github.com/rhysd/github-action-benchmark/commit/895f92f564521597492bd281cbf6c8efd39f628e
 
+## Running CI workflow on a forked repo
+
+Since the benchmark data includes the URL of the repository the tests and examples will fail when the repo is forked. In order to get the ci workflow and all examples running the URL has to be changed
+
+### Change workflows and tests
+
+1. in [`ci_validate_modification.ts`](scripts/ci_validate_modification.ts) in the `validateJSON` function replace `https://github.com/rhysd/github-action-benchmark` with your repo URL (i.e. `https://github.com/<YOU>/github-action-benchmark`)
+2. in all workflow (`.yml`) in `.github/workflows` replace `alert-comment-cc-users: '@rhysd'` with your user
+
+### Adapt past benchmark data to new repo path 
+
+1. checkout the branch `gh-pages` 
+2. in `dev/bench/data.js` replace `https://github.com/rhysd/github-action-benchmark` with your repo URL (i.e. `https://github.com/<YOU>/github-action-benchmark`)
+
+In case you are adding a new tool and wand to run ci, you have to add at least one set of example data to the `data.js` in the gh-pages branch.
+
 ## How to create a new release
 
 1. Run `$ bash scripts/prepare-release.sh v1`

examples/catch2/.gitignore

@@ -0,0 +1 @@
+build/

examples/catch2/CMakeLists.txt

@@ -0,0 +1,26 @@
+cmake_minimum_required(VERSION 3.12)
+project("Catch2_bench")
+
+include(FetchContent)
+
+FetchContent_Declare(
+  catch2
+  GIT_REPOSITORY https://github.com/catchorg/Catch2.git
+  GIT_TAG v2.11.0)
+
+FetchContent_GetProperties(catch2)
+if(NOT catch2_POPULATED)
+  FetchContent_Populate(catch2)
+  add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR})
+endif()
+
+add_executable(${PROJECT_NAME})
+target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp)
+target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
+
+target_compile_options(
+  ${PROJECT_NAME}
+  PRIVATE
+    $<$<CXX_COMPILER_ID:MSVC>:/DCATCH_CONFIG_ENABLE_BENCHMARKING>
+    $<$<OR:$<CXX_COMPILER_ID:GNU>,$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>>:-DCATCH_CONFIG_ENABLE_BENCHMARKING>
+)

examples/catch2/catch2_bench.cpp

@@ -0,0 +1,11 @@
+#include "fib.hpp"
+#define CATCH_CONFIG_MAIN
+#include <catch2/catch.hpp>
+
+TEST_CASE("Fibonacci") {
+
+  // now let's benchmark:
+  BENCHMARK("Fibonacci 20") { return fib(20); };
+
+  BENCHMARK("Fibonacci 25") { return fib(25); };
+}

examples/catch2/fib.hpp

@@ -0,0 +1,11 @@
+#if !defined FIB_HPP_INCLUDED
+#define FIB_HPP_INCLUDED
+
+int fib(int const i) {
+  if (i <= 1) {
+    return 1;
+  }
+  return fib(i - 2) + fib(i - 1);
+}
+
+#endif // FIB_HPP_INCLUDED

src/config.ts

@@ -3,7 +3,7 @@ import { promises as fs } from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 
-export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp';
+export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2';
 export interface Config {
     name: string;
     tool: ToolType;
@@ -22,7 +22,7 @@ export interface Config {
     maxItemsInChart: number | null;
 }
 
-export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp'];
+export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2'];
 const RE_UINT = /^\d+$/;
 
 function validateToolType(tool: string): asserts tool is ToolType {

src/default_index_html.ts

@@ -116,6 +116,7 @@ export const DEFAULT_INDEX_HTML = String.raw`<!DOCTYPE html>
           benchmarkjs: '#f1e05a',
           pytest: '#3572a5',
           googlecpp: '#f34b7d',
+          catch2: '#f34b7d',
           _: '#333333'
         };
 

src/extract.ts

@@ -304,6 +304,82 @@ function extractGoogleCppResult(output: string): BenchmarkResult[] {
     });
 }
 
+function extractCatch2Result(output: string): BenchmarkResult[] {
+    const lines = output.split('\n');
+
+    const ret = [];
+    // Example:
+
+    // benchmark name samples       iterations    estimated <-- Start benchmark section
+    //                mean          low mean      high mean <-- Ignored
+    //                std dev       low std dev   high std dev <-- Ignored
+    // ----------------------------------------------------- <-- Ignored
+    // Fibonacci 20   100           2             8.4318 ms <-- Start actual benchmark
+    //                43.186 us     41.402 us     46.246 us <-- Actual benchmark data
+    //                11.719 us      7.847 us     17.747 us <-- Ignored
+
+    const reTestCaseStart = /^benchmark name +samples +iterations +estimated/;
+    const reBenchmarkStart = /^([a-zA-Z\d ]+) +(\d+) +(\d+) +(\d+(\.\d+)?) (ns|ms|us|s)/;
+    const reBenchmarkValues = /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s)/;
+
+    let benchmarkNr = -1;
+    let testCaseNr = -1;
+
+    let linesSinceBenchmarkStart = -1;
+
+    for (const line of lines) {
+        const m = line.match(reTestCaseStart);
+        if (m !== null) {
+            testCaseNr++;
+        }
+        // no benchmark section found so far, ignore
+        if (testCaseNr < 0) {
+            continue;
+        }
+
+        if (benchmarkNr >= 0) {
+            linesSinceBenchmarkStart++;
+        }
+
+        const benchmarkValueMatch = line.match(reBenchmarkValues);
+        if (benchmarkValueMatch === null && linesSinceBenchmarkStart === 1) {
+            throw new Error(
+                'Retrieved a catch2 benchmark but no values for it\nCatch2 result file is possibly mangled\n\n' + line,
+            );
+        }
+        if (linesSinceBenchmarkStart === 1 && benchmarkValueMatch !== null) {
+            ret[benchmarkNr].value = parseFloat(benchmarkValueMatch[1]);
+            ret[benchmarkNr].unit = benchmarkValueMatch[2];
+        }
+        if (linesSinceBenchmarkStart === 2 && benchmarkValueMatch !== null) {
+            ret[benchmarkNr].range = '+/- ' + benchmarkValueMatch[1].trim();
+        }
+
+        const benchmarkMatch = line.match(reBenchmarkStart);
+        if (benchmarkMatch !== null) {
+            linesSinceBenchmarkStart = 0;
+            benchmarkNr++;
+            ret.push({
+                name: benchmarkMatch[1].trim(),
+                value: 0,
+                range: '',
+                unit: '',
+                extra: benchmarkMatch[2] + ' samples',
+            });
+        }
+    }
+
+    if (
+        ret.every(function(r) {
+            return r.range === '' && r.unit === '';
+        })
+    ) {
+        throw new Error(`Invalid range or unit for catch2 benchmark`);
+    }
+
+    return ret;
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool } = config;
@@ -325,6 +401,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'googlecpp':
             benches = extractGoogleCppResult(output);
             break;
+        case 'catch2':
+            benches = extractCatch2Result(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

src/write.ts

@@ -69,6 +69,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return true;
         case 'googlecpp':
             return false;
+        case 'catch2':
+            return false;
     }
 }
 

test/data/extract/catch2_output.txt

@@ -0,0 +1,32 @@
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Catch2_bench is a Catch v2.11.0 host application.
+Run with -? for options
+
+-------------------------------------------------------------------------------
+Fibonacci
+-------------------------------------------------------------------------------
+/home/doeme/Code/github-action-benchmark/examples/catch2/catch2_bench.cpp:5
+...............................................................................
+
+benchmark name                                  samples       iterations    estimated
+                                                mean          low mean      high mean
+                                                std dev       low std dev   high std dev
+-------------------------------------------------------------------------------
+Fibonacci 20                                              100             2     8.4318 ms 
+                                                    43.186 us     41.402 us     46.246 us 
+                                                    11.719 us      7.847 us     17.747 us 
+
+Fibonacci 25                                              100             1    45.6213 ms 
+                                                   451.183 us    441.654 us    469.296 us 
+                                                    65.064 us     36.524 us    106.192 us 
+
+Fibonacci Integer                                         100             1         45 ms 
+                                                        123 s         441 s    296.123 ns 
+                                                          2 s     36.524 us    106.192 us                                                     
+
+
+===============================================================================
+test cases: 1 | 1 passed
+assertions: - none -
+

test/extract.ts

@@ -54,6 +54,32 @@ describe('extractResult()', function() {
                 },
             ],
         },
+        {
+            tool: 'catch2',
+            expected: [
+                {
+                    name: 'Fibonacci 20',
+                    range: '+/- 11.719',
+                    unit: 'us',
+                    value: 43.186,
+                    extra: '100 samples',
+                },
+                {
+                    name: 'Fibonacci 25',
+                    range: '+/- 65.064',
+                    unit: 'us',
+                    value: 451.183,
+                    extra: '100 samples',
+                },
+                {
+                    name: 'Fibonacci Integer',
+                    range: '+/- 2',
+                    unit: 's',
+                    value: 123,
+                    extra: '100 samples',
+                },
+            ],
+        },
         {
             tool: 'go',
             expected: [