From d7cc768998b98bf463baeccaabcddf9d20ca2744 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 2 Apr 2024 10:29:15 +0200 Subject: [PATCH] improve discord notifications --- .github/workflows/_shared-run-test.yaml | 72 +++++++++++++++ .github/workflows/_shared-run.yaml | 14 +++ .github/workflows/run-manual.yml | 3 + .../validator-proposer-slashing-test.yaml | 90 +++++++++++++++++++ tests.yaml | 24 +++++ 5 files changed, 203 insertions(+) create mode 100644 assertoor-tests/validator-proposer-slashing-test.yaml diff --git a/.github/workflows/_shared-run-test.yaml b/.github/workflows/_shared-run-test.yaml index ad0a7cc..f98fe17 100644 --- a/.github/workflows/_shared-run-test.yaml +++ b/.github/workflows/_shared-run-test.yaml @@ -7,6 +7,10 @@ on: type: string description: "The test identifier." required: true + name: + type: string + description: "The test name." + required: true pairs: type: string description: "The client pairs to use (format: consensus1-execution1,consensus2-execution2,...)." @@ -43,10 +47,20 @@ on: type: string description: "The list of assertoor test files to run." required: true + send_notification: + type: string + description: "Send discord notification on test failure (default: false)." + default: "false" secrets: KUBECONFIG: description: 'Kubernetes config' required: true + DISCORD_HOOK: + description: 'Discord hook' + required: false + GITHUB_TOKEN: + description: 'Github hook' + required: false jobs: run_test: @@ -69,6 +83,7 @@ jobs: fi - name: "Generate kurtosis config with client pairs: ${{ inputs.pairs }}" shell: bash + id: clients run: | clients_file="${{ inputs.clients }}" kurtosis_file="${{ inputs.kurtosis }}" @@ -77,6 +92,8 @@ jobs: mkdir -p ./temp touch ./temp/participants.yaml + echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT + if [ ! -z "$pairs" ]; then resolve_image() { image="$1" @@ -96,6 +113,8 @@ jobs: } echo "participants:" >> ./temp/participants.yaml + touch ./temp/clients_summary.txt + client_idx=0 for pair in $(echo $pairs | tr "," "\n") do @@ -181,6 +200,12 @@ jobs: echo "VC: $vc_type ($vc_image)" fi echo "EL: $el_type ($el_image)" + + participant="${el_type}-${cl_type}" + if [ ! -z "$vc_type" ]; then + participant="${participant}-${vc_type}" + fi + echo "participant $client_idx: $participant" >> ./temp/clients_summary.txt done fi @@ -299,6 +324,53 @@ jobs: run: | kurtosis enclave rm -f assertoor-${{ github.run_id }}-${{ inputs.id }} || true + - name: Send discord notification (on failure) + if: ${{ steps.test_result.outputs.result == 'failure' && inputs.send_notification == 'true' }} + shell: bash + run: | + if [ -z "$DISCORD_HOOK" ]; then + echo "discord hook missing" + exit 1 + fi + + hookjson="{}" + hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')") + #hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')") + + touch ./temp/notification_text.txt + echo "test id: ${{ inputs.id }}" >> ./temp/notification_text.txt + + start_time="${{ steps.clients.outputs.start_time }}" + end_time="$(date +%s)" + runtime=$((end_time-start_time)) + echo "run time: $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> ./temp/notification_text.txt + + cat ./temp/clients_summary.txt >> ./temp/notification_text.txt + + echo "" > ./temp/notification_text.txt + test_status=$( + cat <<"EOF" + ${{ steps.test_result.outputs.test_overview }} + EOF + ) + echo "$test_status" > ./temp/notification_text.txt + + msgjson="{}" + msgjson=$(echo "$msgjson" | jq -c ".title|=$(echo -n "Test Failure: ${{ inputs.name }}" | jq -R -s '.')") + msgjson=$(echo "$msgjson" | jq -c ".description|=$(cat ./temp/notification_text.txt | jq -R -s '.')") + + jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs) + job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | .id') + + msgjson=$(echo "$msgjson" | jq -c ".url|=$(echo -n "https://github.com/${{ github.repository_owner }}/${{ github.repository }}/actions/runs/${{ github.run_id}}/job/${job_id}" | jq -R -s '.')") + hookjson=$(echo "$hookjson" | jq -c ".embeds|=[${msgjson}]") + echo $hookjson | jq + + curl -X POST -H 'Content-Type: application/json' -d "$hookjson" $DISCORD_HOOK + env: + DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Return test result shell: bash run: | diff --git a/.github/workflows/_shared-run.yaml b/.github/workflows/_shared-run.yaml index 4bdb964..a0011e1 100644 --- a/.github/workflows/_shared-run.yaml +++ b/.github/workflows/_shared-run.yaml @@ -6,11 +6,21 @@ on: type: string description: "The test configuration in json format." required: true + send_notification: + type: string + description: "Send discord notification on test failure (default: false)." + default: "false" secrets: RANCHER_TOKEN: description: "The rancher token to login to the rancher server." RANCHER_URL: description: "The rancher url to login to the rancher server." + DISCORD_HOOK: + description: 'Discord hook' + required: false + GITHUB_TOKEN: + description: 'Github hook' + required: false jobs: get_test: name: "Prepare Test" @@ -119,6 +129,7 @@ jobs: uses: ./.github/workflows/_shared-run-test.yaml with: id: "${{ needs.get_test.outputs.id }}-${{ matrix.index }}" + name: "${{ needs.get_test.outputs.name }}" pairs: ${{ matrix.pairs }} worker: ${{ needs.get_test.outputs.worker }} kurtosis: ${{ needs.get_test.outputs.kurtosis }} @@ -128,5 +139,8 @@ jobs: kubeStorageClass: ${{ needs.get_test.outputs.kubernetes_storageclass }} clients: ${{ needs.get_test.outputs.clients }} assertoor_tests: ${{ needs.get_test.outputs.assertoor_tests }} + send_notification: ${{ inputs.send_notification }} secrets: KUBECONFIG: ${{ needs.get_test.outputs.kubeconfig }} + DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/run-manual.yml b/.github/workflows/run-manual.yml index 801ff8e..dde6b12 100644 --- a/.github/workflows/run-manual.yml +++ b/.github/workflows/run-manual.yml @@ -84,6 +84,9 @@ jobs: config: ${{ fromJson(needs.get_tests.outputs.test_configs) }} with: config: ${{ toJSON(matrix.config) }} + send_notification: "true" secrets: RANCHER_URL: ${{ secrets.RANCHER_URL }} RANCHER_TOKEN: ${{ secrets.RANCHER_TOKEN }} + DISCORD_HOOK: ${{ secrets.DISCORD_WEBHOOK }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/assertoor-tests/validator-proposer-slashing-test.yaml b/assertoor-tests/validator-proposer-slashing-test.yaml new file mode 100644 index 0000000..2f3aa41 --- /dev/null +++ b/assertoor-tests/validator-proposer-slashing-test.yaml @@ -0,0 +1,90 @@ +id: validator-proposer-slashing-test +name: "Validator Proposer Slashing Test" +timeout: 1h +config: + #walletPrivkey: "" + validatorMnemonic: "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" + #validatorPairNames: [] +tasks: +- name: check_clients_are_healthy + title: "Check if at least one client is ready" + timeout: 5m + config: + minClientCount: 1 + +# check if all validators propose blocks with proposer slashings +- name: run_task_background + title: "Check if all clients propose blocks with proposer slashings" + config: + onBackgroundComplete: "fail" + backgroundTask: + name: generate_slashings + title: "Generate 1 proposer slashing per slot" + config: + slashingType: "proposer" + startIndex: 500 + limitTotal: 50 + limitPerSlot: 1 + indexCount: 50 + configVars: + mnemonic: "validatorMnemonic" + + foregroundTask: + name: run_task_matrix + title: "Check proposer slashing inclusion with all client pairs" + timeout: 30m + configVars: + matrixValues: "validatorPairNames" + config: + runConcurrent: true + matrixVar: "validatorPairName" + task: + name: check_consensus_block_proposals + title: "Wait for block proposal with proposer slashings from ${validatorPairName}" + configVars: + validatorNamePattern: "validatorPairName" + config: + minProposerSlashingCount: 1 + +- name: "sleep" + title: "Wait 30sec to clear up pending proposer slashings" + config: + duration: 30s + +# check if proposer slashings can be sent via each client pair +- name: run_task_matrix + title: "Check if proposer slashings can be sent via each client pair" + configVars: + matrixValues: "clientPairNames" + config: + runConcurrent: false + matrixVar: "clientPairName" + task: + name: run_tasks + title: "Check if proposer slashings can be sent via ${clientPairName}" + config: + tasks: + - name: run_task_background + title: "Check if proposer slashings can be sent via ${clientPairName}" + config: + onBackgroundComplete: "fail" + backgroundTask: + name: generate_slashings + title: "Generate 1 proposer slashing per slot" + config: + slashingType: "proposer" + startIndex: 550 + limitTotal: 50 + limitPerSlot: 1 + indexCount: 100 + configVars: + mnemonic: "validatorMnemonic" + foregroundTask: + name: check_consensus_block_proposals + title: "Wait for block proposal with proposer slashings" + config: + minProposerSlashingCount: 1 + - name: "sleep" + title: "Wait 30sec to clear up pending proposer slashings" + config: + duration: 30s \ No newline at end of file diff --git a/tests.yaml b/tests.yaml index a99c960..44ee6c1 100644 --- a/tests.yaml +++ b/tests.yaml @@ -142,3 +142,27 @@ tests: - lodestar-geth,lodestar-geth assertoorTests: - assertoor-tests/verkle-conversion-check.yaml + +# Test1 +- id: "proposals" + schedule: false + name: "Test latest clients (proposals only)" + clients: clients/latest.yaml + kurtosis: kurtosis-config/default.yaml + worker: ubuntu-latest + backend: docker + clientPairs: + # CL: [lighthouse,prysm,teku,lodestar,nimbus,grandine] + # EL: [geth,besu,reth,erigon,nethermind] + - lighthouse-reth,lighthouse-besu,grandine-nethermind + #- prysm-reth,lighthouse-erigon,teku-besu + #- prysm-besu,lodestar-besu,nimbus-nethermind + #- lodestar-geth,prysm-erigon,nimbus-besu + #- lighthouse-geth,grandine-besu,nimbus-reth + #- prysm-geth,prysm-nethermind,lodestar-reth + #- lighthouse-nethermind,nimbus-erigon,teku-geth + #- lodestar-erigon,teku-nethermind,grandine-reth + #- grandine-geth,teku-erigon,lodestar-nethermind + #- teku-reth,grandine-erigon,nimbus-geth + assertoorTests: + - assertoor-tests/block-proposal-check.yaml