From fe67380de3163f4b3f925bfc4add8b026c7a4c04 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Thu, 2 Mar 2023 12:02:31 +0100 Subject: [PATCH] [CI] refactor benchmark (#1235) * [WIP][CI] refactor benchmark * change image for benches * disable job * fix docker image * fix ws var * fwd output to file * fix var * rm bench for kusama and westend * set log-level info * mv benchmarks to artifacts * publish results to prom * fix typo, add some echoes * substitute curly brackets * fix naming * return pipeline * rename build jobs --- .gitlab-ci.yml | 125 +++++++----------- .../ci/benchmarks/push_benchmark_results.sh | 30 +++++ 2 files changed, 78 insertions(+), 77 deletions(-) create mode 100755 scripts/ci/benchmarks/push_benchmark_results.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5a7ac765..901568c33 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ variables: DOCKERFILE_DIRECTORY: "./" CI_IMAGE: "quay.io/buildah/stable:v1.27" NODE_IMAGE: "node:18" + BENCHMARK_IMAGE: "paritytech/node-wrk:latest" default: retry: @@ -53,11 +54,11 @@ stages: paths: - ./artifacts/ -.test-refs: &test-refs +.test-refs: &test-refs rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -.test-refs-manual: &test-refs-manual +.test-refs-manual: &test-refs-manual rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: manual @@ -117,27 +118,31 @@ stages: CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io" script: - yarn --immutable - - yarn test:latest-e2e-tests --chain $CHAIN_NAME --local $CHAIN_URL + - echo "Tests for ${CHAIN_URL}" + - yarn test:latest-e2e-tests --log-level info --chain ${CHAIN_NAME} --local ${CHAIN_URL} e2e-westend: <<: *e2e-template variables: CI_IMAGE: $NODE_IMAGE CHAIN_NAME: "westend" + CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io" e2e-kusama: <<: *e2e-template variables: CI_IMAGE: $NODE_IMAGE CHAIN_NAME: "kusama" + CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io" e2e-polkadot: <<: *e2e-template variables: CI_IMAGE: $NODE_IMAGE CHAIN_NAME: "polkadot" + CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io" -build: +build-docker: <<: *dockerize <<: *publish-refs script: @@ -183,7 +188,7 @@ push-docker-image-description: - cd / && sh entrypoint.sh # checks that dockerimage can be built without publishing -build-pr: +build-docker-pr: <<: *dockerize <<: *test-refs script: @@ -215,92 +220,58 @@ deploy-production: - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 when: manual -.benchmark-template: &benchmark-template - stage: benchmark - <<: *collect-artifacts +bench-polkadot: &bench-polkadot + stage: benchmark <<: *kubernetes-env - before_script: [] + <<: *collect-artifacts + <<: *publish-refs variables: - POLKADOT_ADDRESS: "ws://substrate-api-sidecar-benchmark-node.nodes:9944" - BENCHMARK_DURATION: "15m" # Tests duration - BENCHMARK_THREADS: 4 # Total number of threads to use - BENCHMARK_CONNECTIONS: 12 # Total number of HTTP connections to keep open with each thread handling N = connections/threads - BENCHMARK_TIMEOUT: "120s" # Record a timeout if a response is not received within this amount of time - BENCHMARK_OPTS: "--latency" # Additional options, --latency: print detailed latency statistics - BENCHMARK_SCRIPT: "./scripts/ci/benchmarks/lightweight-bench.lua" # https://github.com/wg/wrk/blob/master/SCRIPTING - CI_IMAGE: "paritytech/node-wrk:latest" + CI_IMAGE: $BENCHMARK_IMAGE + CHAIN_NAME: "polkadot" + CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io" script: - - export SAS_SUBSTRATE_URL=$POLKADOT_ADDRESS - - yarn - - yarn build - - yarn run main & - - sleep 10 - - wrk -d${BENCHMARK_DURATION} - -t${BENCHMARK_THREADS} - -c${BENCHMARK_CONNECTIONS} - --timeout ${BENCHMARK_TIMEOUT} - ${BENCHMARK_OPTS} - -s ${BENCHMARK_SCRIPT} "http://127.0.0.1:8080" > result.txt - - cat result.txt - - mkdir artifacts/ - - mv result.txt artifacts/ - tags: - - kubernetes-parity-build - -benchmark: - <<: *benchmark-template - <<: *publish-refs - needs: - - build + - yarn --immutable + - echo "Benchmarks for ${CHAIN_URL}" + - mkdir -p artifacts + - yarn bench --log-level info --ws-url ${CHAIN_URL} + - mv benchmarks.txt artifacts/ # manual step to run benchmarks in PR pipeline -benchmark-manual-pr: - <<: *benchmark-template +bench-polkadot-manual-pr: + stage: benchmark + <<: *bench-polkadot <<: *test-refs-manual - needs: - - build-pr -check-benchmark: - stage: check-benchmark - <<: *publish-refs - <<: *kubernetes-env - <<: *collect-artifacts - needs: - - job: benchmark - artifacts: true - variables: - GITHUB_REPO: "paritytech/substrate-api-sidecar" - CI_IMAGE: "paritytech/benchmarks:latest" - THRESHOLD: 60000 - GITHUB_TOKEN: $GITHUB_PR_TOKEN - script: - - export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}') - - check_single_bench_result -g $GITHUB_REPO - -c $THRESHOLD - -v $RESULT + +# temporary disabled for collecting results +# check-benchmark: +# stage: check-benchmark +# <<: *publish-refs +# <<: *kubernetes-env +# <<: *collect-artifacts +# needs: +# - job: benchmark +# artifacts: true +# variables: +# GITHUB_REPO: "paritytech/substrate-api-sidecar" +# CI_IMAGE: "paritytech/benchmarks:latest" +# THRESHOLD: 60000 +# GITHUB_TOKEN: $GITHUB_PR_TOKEN +# script: +# - export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}') +# - check_single_bench_result -g $GITHUB_REPO +# -c $THRESHOLD +# -v $RESULT push-benchmark: - stage: push-benchmark + stage: push-benchmark <<: *publish-refs <<: *kubernetes-env needs: - - job: check-benchmark + - job: bench-polkadot artifacts: true variables: PROMETHEUS_URL: "https://pushgateway.parity-build.parity.io" CI_IMAGE: "paritytech/benchmarks:latest" script: - - export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}') - - push_bench_result -t common - -p $CI_PROJECT_NAME - -n sidecar - -r $RESULT - -u ms - -s $PROMETHEUS_URL - - push_bench_result -t specific - -p $CI_PROJECT_NAME - -n sidecar - -r $RESULT - -l 'commit="'$CI_COMMIT_SHORT_SHA'"' - -u ms - -s $PROMETHEUS_URL + - scripts/ci/benchmarks/push_benchmark_results.sh artifacts/benchmarks.txt diff --git a/scripts/ci/benchmarks/push_benchmark_results.sh b/scripts/ci/benchmarks/push_benchmark_results.sh new file mode 100755 index 000000000..645a63c56 --- /dev/null +++ b/scripts/ci/benchmarks/push_benchmark_results.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# The script parses benchmarks.txt file and sends benchmarks results to prometheus +# Usage: push_benchmark_results.sh + +RESULT_FILE=$1 + +if [ -z "${RESULT_FILE}" ] +then + echo "Usage: $0 " + exit 1 +fi + + +BENCHMARKS=$(cat ${RESULT_FILE} | grep Result | awk '{print $3}') + +for benchmark in ${BENCHMARKS} +do + # "/accounts/{accountId}/balance-info:" -> "accounts-{accountId}-balance-info" + benchmark_name=$(echo ${benchmark} | cut -d ":" -f1 | sed 's/\///' | sed 's/\//-/g' ) + result=$(cat ${RESULT_FILE} | grep -A 20 ${benchmark} | grep "Avg RequestTime(Latency)" | awk '{print $3}') + unit=${result: -2} + result_value=${result::-2} + echo "--------------------------------------------------------------------------" + echo "Sending metrics for ${benchmark_name} with result: ${result_value} ${unit}" + echo "Sending common metric" + push_bench_result -t common -p $CI_PROJECT_NAME -n ${benchmark_name} -r ${result_value} -u ${unit} -s ${PROMETHEUS_URL} + echo "Sending specific metric" + push_bench_result -t specific -p $CI_PROJECT_NAME -n ${benchmark_name} -r ${result_value} -l 'commit="'${CI_COMMIT_SHORT_SHA}'"' -u ${unit} -s ${PROMETHEUS_URL} + echo "--------------------------------------------------------------------------" +done