Krzysztofpaliga/integration bench

.github/integrationBenchScripts/04_clone.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eo pipefail
+
+GITHUB_RUN_ID=$1
+REPOSITORY_URL=$2
+BRANCH_NAME=$3
+
+# Get the latest temp directory in the home directory
+current_dir="$HOME"/CI_Prover_Benches/"$GITHUB_RUN_ID"
+target_dir="$current_dir"/zkevm-circuits
+
+if [ ! -d "$target_dir" ]; then
+  # Clone the repository into the latest temp directory
+  echo "Cloning the repository $REPOSITORY_URL into the latest temp directory: $current_dir"
+  git clone -q "$REPOSITORY_URL" "$current_dir/zkevm-circuits"
+
+  if [ -n "$BRANCH_NAME" ]; then
+    old_dir=$(pwd)
+    cd "$current_dir/zkevm-circuits" || exit 1
+    git checkout "$BRANCH_NAME"
+    cd "$old_dir" || exit 1
+  fi
+  # Print a message to indicate successful cloning
+  echo "Repository $REPOSITORY_URL cloned successfully into: $current_dir/zkevm-circuits"
+fi

.github/integrationBenchScripts/07_execBench.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#set -eo pipefail
+
+DEGREE=$1
+GITHUB_RUN_ID=$2
+PROVER=$3
+
+export GOROOT="/usr/local/go"
+export GOPATH="$HOME/go"
+export PATH="$GOROOT/bin:$GOPATH/bin:$PATH"
+
+# Get the latest temp directory in the home directory
+current_dir="$HOME"/CI_Prover_Benches/"$GITHUB_RUN_ID"
+
+target_dir="$current_dir/zkevm-circuits"
+
+printf -v _date '%(%Y-%m-%d_%H:%M:%S)T' -1
+
+cd "$target_dir";
+
+mkdir ../results
+logfile="$_date"--"${PROVER}"_bench-"$DEGREE".proverlog
+
+
+current_time=$(date +'%H:%M:%S')
+echo "Current time: $current_time"
+echo "$current_time" > ~/bench_begin
+export RUST_BACKTRACE=1
+
+export LIBCLANG_PATH="/usr/lib/x86_64-linux-gnu/"
+export GETH_L2_URL="http://43.153.26.11:8545/"
+
+if [ "$PROVER" == "Mock" ]; then
+  echo "Running actions for Mock Prover"
+  echo "~/.cargo/bin/cargo test --package integration-tests --test taiko_circuits -- mock_prover::serial_test_evm_circuit_block_anchor_only --exact --nocapture"
+  ~/.cargo/bin/cargo test --package integration-tests --test taiko_circuits -- mock_prover::serial_test_evm_circuit_block_anchor_only --exact --nocapture > "$target_dir/../results/$logfile" 2>&1
+elif [ "$PROVER" == "Real" ]; then
+  echo "Running actions for Real Prover"
+  echo "~/.cargo/bin/cargo test --package integration-tests --test taiko_circuits -- real_prover --nocapture"
+  ~/.cargo/bin/cargo test --package integration-tests --test taiko_circuits -- real_prover --nocapture > "$target_dir/../results/$logfile" 2>&1
+else
+  echo "Unknown PROVER value: $PROVER"
+  exit 1
+fi
+
+RESULT=$?
+echo $RESULT > ../run_result
+echo "exiting 07_exechBench.sh with RESULT $RESULT"
+

.github/integrationBenchScripts/08_processResults.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+#set -eo pipefail
+
+prover=$1
+degree=$2
+
+# Get the latest temp directory in the Triggerers directory
+trigger_results_dir="../../../results"
+mkdir -p "$trigger_results_dir" || true
+
+# Get the latest temp directory in the Provers home directory
+prover_latest_dir=$(ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" <<EOF
+ls -td -- "\$HOME"/CI_Prover_Benches/* | head -1
+EOF
+)
+
+prover_target_dir="$prover_latest_dir/zkevm-circuits"
+prover_results_dir="$prover_latest_dir/results"
+echo "$prover_target_dir"
+
+# Collect results from Prover
+echo "Collecting results from $PROVER_IP:$prover_results_dir to TRIGGER_HOST:$trigger_results_dir"
+scp -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP":"$prover_results_dir"/*proverlog "$trigger_results_dir"/
+
+# Enable bash Environment Variables for Prover
+ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" <<EOF
+echo "PermitUserEnvironment yes" | sudo tee -a /etc/ssh/sshd_config
+sudo service sshd restart
+EOF
+sleep 10
+
+# Collect cpu and memory metrics
+scp -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ./sadf.sh ubuntu@"$PROVER_IP":~/
+# shellcheck disable=SC2086
+ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@$PROVER_IP "bash -s" <<EOF
+BENCH_BEGIN=\$(cat /home/ubuntu/bench_begin)
+echo "Bench began at \$BENCH_BEGIN"
+mv /home/ubuntu/sadf.sh $prover_results_dir/
+cd $prover_results_dir
+./sadf.sh \$BENCH_BEGIN
+EOF
+scp -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP":"$prover_results_dir"/*.stats "$trigger_results_dir"/
+
+# Prepare for and run data processing and db persistence
+
+l=$(echo "$prover" | tr -d '"')
+circuit=$(echo "$l" |  awk '{print $1}')
+time=$(date +%Y-%m-%d_%H-%M-%S)
+test_id=$time-$circuit-$degree-Benchmark
+
+cd "$trigger_results_dir"
+tar -czvf ./"$test_id".tar.gz ./*proverlog ./*.stats
+
+cp ../zkevm-circuits/.github/integrationBenchScripts/reporting*.py .
+sudo cp *proverlog /var/www/www_logs/
+proverlog="http://43.130.90.57/www_logs/"$(ls -t /var/www/www_logs | head -1)
+instance_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*instance.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+instance_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*instance.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+advice_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*advice.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+advice_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*advice.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+fixed_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*fixed.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+fixed_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*fixed.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+lookups_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*lookups.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+lookups_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*lookups.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+equality_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*equality.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+equality_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*equality.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+vanishing_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*vanishing.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+vanishing_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*vanishing.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+multiopen_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*multiopen.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+multiopen_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*multiopen.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+polycomm_commitments=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*polycomm.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $1}')
+polycomm_evaluations=$(ls -t *proverlog | xargs cat | sed 's/\x1B\[[0-9;]*[JKmsu]//g' | egrep -o  "|.*polycomm.*" | egrep -o [0-9]+[\ ]+[0-9]+ | awk '{print $2}')
+
+circuit_cost=$(ls -t *proverlog | xargs cat | egrep -o "CircuitCost.*")
+
+minimum_rows=$(ls -t *proverlog | xargs cat | egrep -o "^minimum_rows.*" | egrep -o "[0-9]+")
+blinding_factors=$(ls -t *proverlog | xargs cat | egrep -o "^blinding_factors.*" | egrep -o "[0-9]+")
+gates_count=$(ls -t *proverlog | xargs cat | egrep -o "^gates count.*" | egrep -o "[0-9]+")
+
+
+sed -i '1i BENCH-PROVER;-1;UTC;LINUX-RESTART	(64 CPU)' mem.stats
+sed -i '1i BENCH-PROVER;-1;UTC;LINUX-RESTART	(64 CPU)' cpu.stats
+python3 reporting_main.py  "$proverlog" "1" "$prover" "$degree" "$test_id" \
+"$instance_commitments" "$instance_evaluations" \
+"$advice_commitments" "$advice_evaluations" \
+"$fixed_commitments" "$fixed_evaluations" \
+"$lookups_commitments" "$lookups_evaluations" \
+"$equality_commitments" "$equality_evaluations" \
+"$vanishing_commitments" "$vanishing_evaluations" \
+"$multiopen_commitments" "$multiopen_evaluations" \
+"$polycomm_commitments" "$polycomm_evaluations" \
+"$circuit_cost" \
+"$minimum_rows" "$blinding_factors" "$gates_count"
+
+ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" <<EOF
+sudo rm -rf $prover_results_dir
+EOF
+

.github/integrationBenchScripts/cloud-tests-local-trigger.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -eo pipefail
+
+cd "$(dirname "$0")" || exit 1
+
+GITHUB_RUN_ID=$1
+REPOSITORY_URL=$2
+BRANCH_NAME=$3
+PROVER=$4
+DEGREE=19
+
+PROVER_INSTANCE=$(cat "$HOME/CI_Github_Trigger/$GITHUB_RUN_ID/prover_instance")
+echo "Prover instance at trigger: $PROVER_INSTANCE"
+
+export PROVER_IP=$(tccli cvm DescribeInstances --InstanceIds "[\"$PROVER_INSTANCE\"]" | grep -A 1 PublicIpAddress | egrep -o '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')
+echo "Prover IP: $PROVER_IP"
+
+rm ~/.ssh/known_hosts*
+
+prepare_env() {
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- <../weeklyBenchScripts/00_installGo.sh
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- <../weeklyBenchScripts/00_installRust.sh
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- <../weeklyBenchScripts/01_installDeps.sh
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- <../weeklyBenchScripts/02_setup.sh
+}
+
+prepare_repo() {
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- "$GITHUB_RUN_ID" <../weeklyBenchScripts/03_prepareProver.sh
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- "$GITHUB_RUN_ID" "$REPOSITORY_URL" "$BRANCH_NAME" <../integrationBenchScripts/04_clone.sh
+#  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- "$GITHUB_RUN_ID" <../weeklyBenchScripts/05_build.sh
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- <../weeklyBenchScripts/06_rsSysstat.sh &
+  sleep 5
+
+  ssh -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP" "bash -s" -- "$DEGREE" "$GITHUB_RUN_ID" "$PROVER" <../integrationBenchScripts/07_execBench.sh
+  declare -g RESULT=$?
+  chmod u+x ../integrationBenchScripts/08_processResults.sh
+  ../integrationBenchScripts/08_processResults.sh "$PROVER" "$DEGREE"
+}
+
+prepare_env
+prepare_repo
+
+# Rationale: for some (?) reason the very last ssh session is not being closed (07_execBench.sh)
+kill_ssh() {
+  sleep 30
+  # Get the list of process IDs with the given IP address and Option which should normally be used only by a script
+  pids=$(ps aux | grep "StrictHostKeyChecking=no" | grep "$PROVER_IP" | grep -v "grep" | awk '{print $2}')
+
+  # Loop through the process IDs and kill them
+  for pid in $pids; do
+    echo "Killing process with PID: $pid"
+    kill "$pid"
+  done
+}
+
+kill_ssh &
+
+scp -i ~/.ssh/bench.pem -o StrictHostKeyChecking=no ubuntu@"$PROVER_IP":"$HOME"/CI_Prover_Benches/"$GITHUB_RUN_ID"/run_result ../../../
+RESULT=$(cat ../../../run_result)
+echo "exiting cloud-tests-local-trigger with RESULT $RESULT"
+exit "$RESULT"

.github/integrationBenchScripts/cloud-tests-trigger.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -eo pipefail
+
+GITHUB_RUN_ID=$1
+REPOSITORY_URL=$2
+BRANCH_NAME=$3
+PROVER=$4
+
+ensure_git_installed() {
+  if ! command -v git &>/dev/null; then
+    echo "Git is not installed. Installing..."
+    sudo apt update
+    sudo apt install -y git
+  else
+    echo "Git is already installed."
+  fi
+}
+
+ensure_git_installed
+
+clone_zkevm-circuits() {
+  git clone -q https://github.com/taikoxyz/zkevm-circuits.git
+  cd zkevm-circuits || exit 1
+  echo "Cloned zkevm-circuits"
+}
+
+directory_name="$HOME/CI_Github_Trigger/$GITHUB_RUN_ID"
+cd "$directory_name" || exit 1
+
+
+clone_zkevm-circuits
+
+cd .github/integrationBenchScripts || exit 1
+chmod u+x cloud-tests-local-trigger.sh
+echo "Triggering ./cloud-tests-local-trigger.sh with $GITHUB_RUN_ID $REPOSITORY_URL $BRANCH_NAME $PROVER"
+./cloud-tests-local-trigger.sh "$GITHUB_RUN_ID" "$REPOSITORY_URL" "$BRANCH_NAME" "$PROVER"
+RESULT=$?
+echo "exiting cloud-tests-trigger with result $RESULT"
+exit $RESULT

.github/integrationBenchScripts/github-action-trigger.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eo pipefail
+
+echo "Triggering cloud-tests-trigger.sh with $GITHUB_RUN_ID $REPOSITORY_URL $BRANCH_NAME $PROVER"
+sshpass -p "$BENCH_RESULTS_PASS" ssh -o StrictHostKeyChecking=no ubuntu@43.130.90.57 "bash -s" -- "$GITHUB_RUN_ID" "$REPOSITORY_URL" "$BRANCH_NAME" "$PROVER" <cloud-tests-trigger.sh
+RESULT=$?
+echo "exiting github-action-trigger with RESULT=$RESULT"
+exit $RESULT

.github/integrationBenchScripts/reporting_main.py

@@ -0,0 +1,66 @@
+import argparse, json
+from pprint import pprint
+import reporting_modules as rmod
+
+env = json.load(open('/etc/taiko/env.json'))
+cstats = './cpu.stats'
+mstats = './mem.stats'
+
+def main():
+    parser = argparse.ArgumentParser(
+                    prog = 'BenchmarkResults',
+                    usage = 'python3 reporting_main.py logfile 13 EVM 19',
+                    description = 'Writes circuit benchmark results to postgresql, uploads logfiles to s3 bucket',
+                    )
+    parser.add_argument('logfile')
+    parser.add_argument('pr')
+    parser.add_argument('prover')
+    parser.add_argument('degree')
+    parser.add_argument('test_id')
+    parser.add_argument('instance_commitments')
+    parser.add_argument('instance_evaluations')
+    parser.add_argument('advice_commitments')
+    parser.add_argument('advice_evaluations')
+    parser.add_argument('fixed_commitments')
+    parser.add_argument('fixed_evaluations')
+    parser.add_argument('lookups_commitments')
+    parser.add_argument('lookups_evaluations')
+    parser.add_argument('equality_commitments')
+    parser.add_argument('equality_evaluations')
+    parser.add_argument('vanishing_commitments')
+    parser.add_argument('vanishing_evaluations')
+    parser.add_argument('multiopen_commitments')
+    parser.add_argument('multiopen_evaluations')
+    parser.add_argument('polycom_commitments')
+    parser.add_argument('polycom_evaluations')
+    parser.add_argument('circuit_cost')
+    parser.add_argument('minimum_rows')
+    parser.add_argument('blinding_factors')
+    parser.add_argument('gates_count')
+    args = parser.parse_args()
+    logfile, pr, prover, degree, test_id = (args.logfile, args.pr, args.prover, args.degree, args.test_id)
+    instance_commitments, instance_evaluations = (args.instance_commitments, args.instance_evaluations)
+    advice_commitments, advice_evaluations = (args.advice_commitments, args.advice_evaluations)
+    fixed_commitments, fixed_evaluations  = (args.fixed_commitments, args.fixed_evaluations)
+    lookups_commitments, lookups_evaluations = (args.lookups_commitments, args.lookups_evaluations)
+    equality_commitments, equality_evaluations = (args.equality_commitments, args.equality_evaluations)
+    vanishing_commitments, vanishing_evaluations = (args.vanishing_commitments, args.vanishing_evaluations)
+    multiopen_commitments, multiopen_evaluations = (args.multiopen_commitments, args.multiopen_evaluations)
+    polycom_commitments, polycom_evaluations = (args.polycom_commitments, args.polycom_evaluations)
+    circuit_cost = args.circuit_cost
+    minimum_rows, blinding_factors, gates_count = (args.minimum_rows, args.blinding_factors, args.gates_count)
+    test_result = rmod.log_processor(pr, degree)
+    cpustats, memstats, sysstat = rmod.calc_stats(cstats,mstats)
+    data = rmod.prepare_result_dataframe(prover, degree, logfile, test_result, sysstat, env, test_id, instance_commitments, instance_evaluations, advice_commitments, advice_evaluations, fixed_commitments, fixed_evaluations, lookups_commitments, lookups_evaluations, equality_commitments, equality_evaluations, vanishing_commitments, vanishing_evaluations, multiopen_commitments, multiopen_evaluations, polycom_commitments, polycom_evaluations, circuit_cost, minimum_rows, blinding_factors, gates_count)
+    table = 'testresults_integration_benchmark'
+    engine = rmod.pgsql_engine(env['db'])
+    data.to_sql(table,engine,if_exists='append')
+    ms = rmod.write_mem_time(engine,memstats, test_id)
+    cs = rmod.write_cpuall_time(engine,cpustats, test_id)
+
+    # url = f'{env["grafana_dashboard_prefix"]}{test_id}'.replace(" ", "")
+    # print(f'Test Result: {url}')
+
+if __name__ == '__main__':
+    main()
+