publish_benchmarks.yml

---
name: Publish Performance Testing Benchmarks

on:  # yamllint disable-line rule:truthy
  push:
    branches:
      - master
  workflow_dispatch:
  pull_request:

jobs:
  build-and-publish-benchmarks:
    strategy:
      matrix:
        os: ["ubuntu-latest", "macos-latest"]
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        library-type: ["pure_python", "c_library"]

    name: Performance testing for Python ${{ matrix.python-version }} (${{ matrix.os }}) [${{ matrix.library-type }}]
    runs-on: ${{ matrix.os }}
    permissions:
      pull-requests: write
      contents: write
      pages: write

    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          # yamllint disable-line rule:braces
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry
        run: |
          pip install --constraint=.github/workflows/constraints.txt poetry
          poetry --version

      - name: Configure Poetry
        run: |
          poetry config cache-dir "${GITHUB_WORKSPACE}/.cache/pypoetry"
          poetry config virtualenvs.in-project true
          poetry config --list

      - name: Install Tox
        run: |
          pip install --constraint=.github/workflows/constraints.txt tox
          tox --version

      - name: Load cached tox testenv(s) (if they exist)
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: |
            .tox
          key: tox-${{ github.workflow }}-${{ github.job }}-${{ runner.os }}-CPython${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: Run benchmarks for ${{ matrix.python-version }}
        id: performance-testing
        uses: nick-invision/retry@v3
        with:
          timeout_minutes: 20
          max_attempts: 3
          shell: bash
          command: |
            TOX_COMPATIBLE_VERSION=$(echo ${{ matrix.python-version }} | tr -d '.')
            TOX_TESTENV="py${TOX_COMPATIBLE_VERSION}-benchmark-${{ matrix.library-type }}"
            make "tox-${TOX_TESTENV}"

            BENCHMARK_FILE=$(find .benchmarks -maxdepth 2 -type f -name "*.json" | sort  --version-sort --reverse | tail -n 1)
            echo "benchmark_file_path=${BENCHMARK_FILE}" >> $GITHUB_OUTPUT

            ENV_DIR_NAME=$(basename "${BENCHMARK_FILE%/*}" | tr '-' '/')
            NAMESPACED_TARGET_OUTPUT_DIR="${ENV_DIR_NAME}/${{ matrix.library-type }}"
            echo "target_output_dir=${NAMESPACED_TARGET_OUTPUT_DIR}" >> $GITHUB_OUTPUT

      - name: Store the environment-specific benchmarks
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-data-files-${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.library-type }}
          path: .benchmarks

      - name: Reset Poetry lockfile for Checkout
        run: |
          git checkout poetry.lock || echo "Skipping step"

      - name: Publish benchmarks to GH pages
        uses: TeoZosa/github-action-benchmark@v1.8.2
        if: github.ref == 'refs/heads/master'
        with:
          tool: 'pytest'
          benchmark-data-dir-path: dev/bench/${{ steps.performance-testing.outputs.target_output_dir }}
          # yamllint disable-line rule:braces
          output-file-path: ${{ steps.performance-testing.outputs.benchmark_file_path }}
          # yamllint disable-line rule:braces
          github-token: ${{ secrets.GITHUB_TOKEN }}
          # Push and deploy GitHub pages branch automatically
          auto-push: true

  # Plot aggregate benchmarks table ----------------------
  plot-benchmarks-table:
    name: Plot aggregated benchmarks table
    runs-on: ubuntu-latest
    needs:
      - build-and-publish-benchmarks
    env:
      BENCHMARK_OPTS: "--name=normal --sort=name --group-by=name --histogram"
      PY_COLORS: 1
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Upgrade pip
        run: |
          pip install --constraint=.github/workflows/constraints.txt pip
          pip --version

      - name: Install pytest-benchmark
        run: |
          pip install pytest-benchmark[histogram]

      - name: Download the benchmarks
        uses: actions/download-artifact@v4
        with:
          pattern: benchmark-data-files*
          path: .benchmarks
          merge-multiple: true

      - name: List benchmark data files
        run: |
          find .benchmarks

      - name: Plot aggregate results tables (by Python interpreter)
        run: |
          # Disable '\n' field splitting for proper downstream command substitutions
          IFS=' '
          BENCHMARK_DATA_DIRS=$(ls .benchmarks | grep -v "^.*bit$" | sed 's/\(.*\)/\1\/*/')

          echo "**By Python versions**"
          for MINOR_VER in 7 8 9;
          do
            PY_VER=Cpython3.${MINOR_VER}
            PY_VER_BENCH_DIRS=$(echo ${BENCHMARK_DATA_DIRS} | grep "^.*${PY_VER}.*$" | tr '\n' ' ')
            echo "${PY_VER} benchmark data directories: ${PY_VER_BENCH_DIRS}"

            pytest-benchmark compare ${PY_VER_BENCH_DIRS} ${{ env.BENCHMARK_OPTS }}
          done

      - name: Plot aggregate results tables (by platform)
        run: |
          # Disable '\n' field splitting for proper downstream command substitutions
          IFS=' '
          BENCHMARK_DATA_DIRS=$(ls .benchmarks | grep -v "^.*bit$" | sed 's/\(.*\)/\1\/*/')

          for SYS_PLATFORM in Linux Darwin Win32;
          do
            SYS_PLATFORM_BENCH_DIRS=$(echo ${BENCHMARK_DATA_DIRS} | grep  "^${SYS_PLATFORM}.*$" | tr '\n' ' ')
            echo "${SYS_PLATFORM} benchmark data directories: ${SYS_PLATFORM_BENCH_DIRS}"

            pytest-benchmark compare ${SYS_PLATFORM_BENCH_DIRS} ${{ env.BENCHMARK_OPTS }}
          done

      - name: Plot aggregate results tables (platform x Python interpreter)
        run: |
          # Disable '\n' field splitting for proper downstream command substitutions
          IFS=' '
          BENCHMARK_DATA_DIRS=$(ls .benchmarks | grep -v "^.*bit$" | sed 's/\(.*\)/\1\/*/')

          echo "**Pure Python vs C Library (platform x Python interpreter)**"
          for DIR in $(echo ${BENCHMARK_DATA_DIRS} | tr '\n' ' ');
          do
            echo "Platform x Python intepreter: ${DIR}"
            pytest-benchmark compare ${DIR} ${{ env.BENCHMARK_OPTS }}
          done

      - name: Plot aggregate results tables (by library type)
        run: |
          # Disable '\n' field splitting for proper downstream command substitutions
          IFS=' '
          BENCHMARK_DATA_DIRS=$(ls .benchmarks | grep -v "^.*bit$" | sed 's/\(.*\)/\1\/*/')

          echo "**Pure Python (All)**"
          pytest-benchmark compare pure_python ${{ env.BENCHMARK_OPTS }}

          echo "**C Library**"
          pytest-benchmark compare c_library ${{ env.BENCHMARK_OPTS }}

          echo "**Pure Python vs. C library (Aggregate)**"
          pytest-benchmark compare pure_python c_library ${{ env.BENCHMARK_OPTS }}