Skip to content

Commit

Permalink
[CI] refactor benchmark (#1235)
Browse files Browse the repository at this point in the history 
* [WIP][CI] refactor benchmark

* change image for benches

* disable job

* fix docker image

* fix ws var

* fwd output to file

* fix var

* rm bench for kusama and westend

* set log-level info

* mv benchmarks to artifacts

* publish results to prom

* fix typo, add some echoes

* substitute curly brackets

* fix naming

* return pipeline

* rename build jobs
  • Loading branch information
@alvicsam
alvicsam committed Mar 2, 2023
1 parent 97c1ca6 commit fe67380
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 77 deletions.
125 changes: 48 additions & 77 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ variables:
DOCKERFILE_DIRECTORY: "./"
CI_IMAGE: "quay.io/buildah/stable:v1.27"
NODE_IMAGE: "node:18"
BENCHMARK_IMAGE: "paritytech/node-wrk:latest"

default:
retry:
Expand Down Expand Up @@ -53,11 +54,11 @@ stages:
paths:
- ./artifacts/

.test-refs: &test-refs
.test-refs: &test-refs
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs

.test-refs-manual: &test-refs-manual
.test-refs-manual: &test-refs-manual
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: manual
Expand Down Expand Up @@ -117,27 +118,31 @@ stages:
CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io"
script:
- yarn --immutable
- yarn test:latest-e2e-tests --chain $CHAIN_NAME --local $CHAIN_URL
- echo "Tests for ${CHAIN_URL}"
- yarn test:latest-e2e-tests --log-level info --chain ${CHAIN_NAME} --local ${CHAIN_URL}

e2e-westend:
<<: *e2e-template
variables:
CI_IMAGE: $NODE_IMAGE
CHAIN_NAME: "westend"
CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io"

e2e-kusama:
<<: *e2e-template
variables:
CI_IMAGE: $NODE_IMAGE
CHAIN_NAME: "kusama"
CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io"

e2e-polkadot:
<<: *e2e-template
variables:
CI_IMAGE: $NODE_IMAGE
CHAIN_NAME: "polkadot"
CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io"

build:
build-docker:
<<: *dockerize
<<: *publish-refs
script:
Expand Down Expand Up @@ -183,7 +188,7 @@ push-docker-image-description:
- cd / && sh entrypoint.sh

# checks that dockerimage can be built without publishing
build-pr:
build-docker-pr:
<<: *dockerize
<<: *test-refs
script:
Expand Down Expand Up @@ -215,92 +220,58 @@ deploy-production:
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
when: manual

.benchmark-template: &benchmark-template
stage: benchmark
<<: *collect-artifacts
bench-polkadot: &bench-polkadot
stage: benchmark
<<: *kubernetes-env
before_script: []
<<: *collect-artifacts
<<: *publish-refs
variables:
POLKADOT_ADDRESS: "ws://substrate-api-sidecar-benchmark-node.nodes:9944"
BENCHMARK_DURATION: "15m" # Tests duration
BENCHMARK_THREADS: 4 # Total number of threads to use
BENCHMARK_CONNECTIONS: 12 # Total number of HTTP connections to keep open with each thread handling N = connections/threads
BENCHMARK_TIMEOUT: "120s" # Record a timeout if a response is not received within this amount of time
BENCHMARK_OPTS: "--latency" # Additional options, --latency: print detailed latency statistics
BENCHMARK_SCRIPT: "./scripts/ci/benchmarks/lightweight-bench.lua" # https://github.com/wg/wrk/blob/master/SCRIPTING
CI_IMAGE: "paritytech/node-wrk:latest"
CI_IMAGE: $BENCHMARK_IMAGE
CHAIN_NAME: "polkadot"
CHAIN_URL: "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io"
script:
- export SAS_SUBSTRATE_URL=$POLKADOT_ADDRESS
- yarn
- yarn build
- yarn run main &
- sleep 10
- wrk -d${BENCHMARK_DURATION}
-t${BENCHMARK_THREADS}
-c${BENCHMARK_CONNECTIONS}
--timeout ${BENCHMARK_TIMEOUT}
${BENCHMARK_OPTS}
-s ${BENCHMARK_SCRIPT} "http://127.0.0.1:8080" > result.txt
- cat result.txt
- mkdir artifacts/
- mv result.txt artifacts/
tags:
- kubernetes-parity-build

benchmark:
<<: *benchmark-template
<<: *publish-refs
needs:
- build
- yarn --immutable
- echo "Benchmarks for ${CHAIN_URL}"
- mkdir -p artifacts
- yarn bench --log-level info --ws-url ${CHAIN_URL}
- mv benchmarks.txt artifacts/

# manual step to run benchmarks in PR pipeline
benchmark-manual-pr:
<<: *benchmark-template
bench-polkadot-manual-pr:
stage: benchmark
<<: *bench-polkadot
<<: *test-refs-manual
needs:
- build-pr

check-benchmark:
stage: check-benchmark
<<: *publish-refs
<<: *kubernetes-env
<<: *collect-artifacts
needs:
- job: benchmark
artifacts: true
variables:
GITHUB_REPO: "paritytech/substrate-api-sidecar"
CI_IMAGE: "paritytech/benchmarks:latest"
THRESHOLD: 60000
GITHUB_TOKEN: $GITHUB_PR_TOKEN
script:
- export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}')
- check_single_bench_result -g $GITHUB_REPO
-c $THRESHOLD
-v $RESULT

# temporary disabled for collecting results
# check-benchmark:
# stage: check-benchmark
# <<: *publish-refs
# <<: *kubernetes-env
# <<: *collect-artifacts
# needs:
# - job: benchmark
# artifacts: true
# variables:
# GITHUB_REPO: "paritytech/substrate-api-sidecar"
# CI_IMAGE: "paritytech/benchmarks:latest"
# THRESHOLD: 60000
# GITHUB_TOKEN: $GITHUB_PR_TOKEN
# script:
# - export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}')
# - check_single_bench_result -g $GITHUB_REPO
# -c $THRESHOLD
# -v $RESULT

push-benchmark:
stage: push-benchmark
stage: push-benchmark
<<: *publish-refs
<<: *kubernetes-env
needs:
- job: check-benchmark
- job: bench-polkadot
artifacts: true
variables:
PROMETHEUS_URL: "https://pushgateway.parity-build.parity.io"
CI_IMAGE: "paritytech/benchmarks:latest"
script:
- export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}')
- push_bench_result -t common
-p $CI_PROJECT_NAME
-n sidecar
-r $RESULT
-u ms
-s $PROMETHEUS_URL
- push_bench_result -t specific
-p $CI_PROJECT_NAME
-n sidecar
-r $RESULT
-l 'commit="'$CI_COMMIT_SHORT_SHA'"'
-u ms
-s $PROMETHEUS_URL
- scripts/ci/benchmarks/push_benchmark_results.sh artifacts/benchmarks.txt
30 changes: 30 additions & 0 deletions scripts/ci/benchmarks/push_benchmark_results.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
# The script parses benchmarks.txt file and sends benchmarks results to prometheus
# Usage: push_benchmark_results.sh <path_to_benchmarks.txt>

RESULT_FILE=$1

if [ -z "${RESULT_FILE}" ]
then
echo "Usage: $0 <path_to_benchmarks.txt>"
exit 1
fi


BENCHMARKS=$(cat ${RESULT_FILE} | grep Result | awk '{print $3}')

for benchmark in ${BENCHMARKS}
do
# "/accounts/{accountId}/balance-info:" -> "accounts-{accountId}-balance-info"
benchmark_name=$(echo ${benchmark} | cut -d ":" -f1 | sed 's/\///' | sed 's/\//-/g' )
result=$(cat ${RESULT_FILE} | grep -A 20 ${benchmark} | grep "Avg RequestTime(Latency)" | awk '{print $3}')
unit=${result: -2}
result_value=${result::-2}
echo "--------------------------------------------------------------------------"
echo "Sending metrics for ${benchmark_name} with result: ${result_value} ${unit}"
echo "Sending common metric"
push_bench_result -t common -p $CI_PROJECT_NAME -n ${benchmark_name} -r ${result_value} -u ${unit} -s ${PROMETHEUS_URL}
echo "Sending specific metric"
push_bench_result -t specific -p $CI_PROJECT_NAME -n ${benchmark_name} -r ${result_value} -l 'commit="'${CI_COMMIT_SHORT_SHA}'"' -u ${unit} -s ${PROMETHEUS_URL}
echo "--------------------------------------------------------------------------"
done

0 comments on commit fe67380

Please sign in to comment.