improve discord notifications

.github/workflows/_shared-run-test.yaml

@@ -7,6 +7,10 @@ on:
         type: string
         description: "The test identifier."
         required: true
+      name:
+        type: string
+        description: "The test name."
+        required: true
       pairs:
         type: string
         description: "The client pairs to use (format: consensus1-execution1,consensus2-execution2,...)."
@@ -43,10 +47,20 @@ on:
         type: string
         description: "The list of assertoor test files to run."
         required: true
+      send_notification:
+        type: string
+        description: "Send discord notification on test failure (default: false)."
+        default: "false"
     secrets:
       KUBECONFIG:
         description: 'Kubernetes config'
         required: true
+      DISCORD_HOOK:
+        description: 'Discord hook'
+        required: false
+      GITHUB_TOKEN:
+        description: 'Github hook'
+        required: false
 
 jobs:
   run_test:
@@ -69,6 +83,7 @@ jobs:
         fi
     - name: "Generate kurtosis config with client pairs: ${{ inputs.pairs }}"
       shell: bash
+      id: clients
       run: |
         clients_file="${{ inputs.clients }}"
         kurtosis_file="${{ inputs.kurtosis }}"
@@ -77,6 +92,8 @@ jobs:
         mkdir -p ./temp
         touch ./temp/participants.yaml
 
+        echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
+
         if [ ! -z "$pairs" ]; then
           resolve_image() {
             image="$1"
@@ -96,6 +113,8 @@ jobs:
           }
 
           echo "participants:" >> ./temp/participants.yaml
+          touch ./temp/clients_summary.txt
+
           client_idx=0
           for pair in $(echo $pairs | tr "," "\n")
           do
@@ -181,6 +200,12 @@ jobs:
               echo "VC: $vc_type  ($vc_image)"
             fi
             echo "EL: $el_type  ($el_image)"
+
+            participant="${el_type}-${cl_type}"
+            if [ ! -z "$vc_type" ]; then
+              participant="${participant}-${vc_type}"
+            fi
+            echo "participant $client_idx: $participant" >> ./temp/clients_summary.txt
           done
         fi
 
@@ -299,6 +324,53 @@ jobs:
       run: |
         kurtosis enclave rm -f assertoor-${{ github.run_id }}-${{ inputs.id }} || true
     
+    - name: Send discord notification (on failure)
+      if: ${{ steps.test_result.outputs.result == 'failure' && inputs.send_notification == 'true' }}
+      shell: bash
+      run: |
+        if [ -z "$DISCORD_HOOK" ]; then
+          echo "discord hook missing"
+          exit 1
+        fi
+
+        hookjson="{}"
+        hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')")
+        #hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')")
+
+        touch ./temp/notification_text.txt
+        echo "test id: ${{ inputs.id }}" >> ./temp/notification_text.txt
+
+        start_time="${{ steps.clients.outputs.start_time }}"
+        end_time="$(date +%s)"
+        runtime=$((end_time-start_time))
+        echo "run time: $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> ./temp/notification_text.txt
+
+        cat ./temp/clients_summary.txt >> ./temp/notification_text.txt
+
+        echo "" > ./temp/notification_text.txt
+        test_status=$(
+        cat <<"EOF"
+        ${{ steps.test_result.outputs.test_overview }}
+        EOF
+        )
+        echo "$test_status" > ./temp/notification_text.txt
+
+        msgjson="{}"
+        msgjson=$(echo "$msgjson" | jq -c ".title|=$(echo -n "Test Failure: ${{ inputs.name }}" | jq -R -s '.')")
+        msgjson=$(echo "$msgjson" | jq -c ".description|=$(cat ./temp/notification_text.txt | jq -R -s '.')")
+
+        jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs)
+        job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | .id')
+
+        msgjson=$(echo "$msgjson" | jq -c ".url|=$(echo -n "https://github.com/${{ github.repository_owner }}/${{ github.repository }}/actions/runs/${{ github.run_id}}/job/${job_id}" | jq -R -s '.')")
+        hookjson=$(echo "$hookjson" | jq -c ".embeds|=[${msgjson}]")
+        echo $hookjson | jq
+
+        curl -X POST -H 'Content-Type: application/json' -d "$hookjson" $DISCORD_HOOK
+      env:
+        DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
     - name: Return test result
       shell: bash
       run: |

.github/workflows/_shared-run.yaml

@@ -6,11 +6,21 @@ on:
         type: string
         description: "The test configuration in json format."
         required: true
+      send_notification:
+        type: string
+        description: "Send discord notification on test failure (default: false)."
+        default: "false"
     secrets:
       RANCHER_TOKEN:
         description: "The rancher token to login to the rancher server."
       RANCHER_URL:
         description: "The rancher url to login to the rancher server."
+      DISCORD_HOOK:
+        description: 'Discord hook'
+        required: false
+      GITHUB_TOKEN:
+        description: 'Github hook'
+        required: false
 jobs:
   get_test:
     name: "Prepare Test"
@@ -119,6 +129,7 @@ jobs:
     uses: ./.github/workflows/_shared-run-test.yaml
     with:
       id: "${{ needs.get_test.outputs.id }}-${{ matrix.index }}"
+      name: "${{ needs.get_test.outputs.name }}"
       pairs: ${{ matrix.pairs }}
       worker: ${{ needs.get_test.outputs.worker }}
       kurtosis: ${{ needs.get_test.outputs.kurtosis }}
@@ -128,5 +139,8 @@ jobs:
       kubeStorageClass: ${{ needs.get_test.outputs.kubernetes_storageclass }}
       clients: ${{ needs.get_test.outputs.clients }}
       assertoor_tests: ${{ needs.get_test.outputs.assertoor_tests }}
+      send_notification: ${{ inputs.send_notification }}
     secrets:
       KUBECONFIG: ${{ needs.get_test.outputs.kubeconfig }}
+      DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/run-manual.yml

@@ -84,6 +84,9 @@ jobs:
         config: ${{ fromJson(needs.get_tests.outputs.test_configs) }}
     with:
       config: ${{ toJSON(matrix.config) }}
+      send_notification: "true"
     secrets:
       RANCHER_URL: ${{ secrets.RANCHER_URL }}
       RANCHER_TOKEN: ${{ secrets.RANCHER_TOKEN }}
+      DISCORD_HOOK: ${{ secrets.DISCORD_WEBHOOK }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

assertoor-tests/validator-proposer-slashing-test.yaml

@@ -0,0 +1,90 @@
+id: validator-proposer-slashing-test
+name: "Validator Proposer Slashing Test"
+timeout: 1h
+config:
+  #walletPrivkey: ""
+  validatorMnemonic: "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete"
+  #validatorPairNames: []
+tasks:
+- name: check_clients_are_healthy
+  title: "Check if at least one client is ready"
+  timeout: 5m
+  config:
+    minClientCount: 1
+
+# check if all validators propose blocks with proposer slashings
+- name: run_task_background
+  title: "Check if all clients propose blocks with proposer slashings"
+  config:
+    onBackgroundComplete: "fail"
+    backgroundTask:
+      name: generate_slashings
+      title: "Generate 1 proposer slashing per slot"
+      config:
+        slashingType: "proposer"
+        startIndex: 500
+        limitTotal: 50
+        limitPerSlot: 1
+        indexCount: 50
+      configVars:
+        mnemonic: "validatorMnemonic"
+
+    foregroundTask:
+      name: run_task_matrix
+      title: "Check proposer slashing inclusion with all client pairs"
+      timeout: 30m
+      configVars:
+        matrixValues: "validatorPairNames"
+      config:
+        runConcurrent: true
+        matrixVar: "validatorPairName"
+        task:
+          name: check_consensus_block_proposals
+          title: "Wait for block proposal with proposer slashings from ${validatorPairName}"
+          configVars:
+            validatorNamePattern: "validatorPairName"
+          config:
+            minProposerSlashingCount: 1
+
+- name: "sleep"
+  title: "Wait 30sec to clear up pending proposer slashings"
+  config:
+    duration: 30s
+
+# check if proposer slashings can be sent via each client pair
+- name: run_task_matrix
+  title: "Check if proposer slashings can be sent via each client pair"
+  configVars:
+    matrixValues: "clientPairNames"
+  config:
+    runConcurrent: false
+    matrixVar: "clientPairName"
+    task:
+      name: run_tasks
+      title: "Check if proposer slashings can be sent via ${clientPairName}"
+      config:
+        tasks:
+        - name: run_task_background
+          title: "Check if proposer slashings can be sent via ${clientPairName}"
+          config:
+            onBackgroundComplete: "fail"
+            backgroundTask:
+              name: generate_slashings
+              title: "Generate 1 proposer slashing per slot"
+              config:
+                slashingType: "proposer"
+                startIndex: 550
+                limitTotal: 50
+                limitPerSlot: 1
+                indexCount: 100
+              configVars:
+                mnemonic: "validatorMnemonic"
+            foregroundTask:
+              name: check_consensus_block_proposals
+              title: "Wait for block proposal with proposer slashings"
+              config:
+                minProposerSlashingCount: 1
+        - name: "sleep"
+          title: "Wait 30sec to clear up pending proposer slashings"
+          config:
+            duration: 30s

tests.yaml

@@ -142,3 +142,27 @@ tests:
   - lodestar-geth,lodestar-geth
   assertoorTests:
   - assertoor-tests/verkle-conversion-check.yaml
+
+# Test1
+- id: "proposals"
+  schedule: false
+  name: "Test latest clients (proposals only)"
+  clients: clients/latest.yaml
+  kurtosis: kurtosis-config/default.yaml
+  worker: ubuntu-latest
+  backend: docker
+  clientPairs:
+  # CL: [lighthouse,prysm,teku,lodestar,nimbus,grandine]
+  # EL: [geth,besu,reth,erigon,nethermind]
+  - lighthouse-reth,lighthouse-besu,grandine-nethermind
+  #- prysm-reth,lighthouse-erigon,teku-besu
+  #- prysm-besu,lodestar-besu,nimbus-nethermind
+  #- lodestar-geth,prysm-erigon,nimbus-besu
+  #- lighthouse-geth,grandine-besu,nimbus-reth
+  #- prysm-geth,prysm-nethermind,lodestar-reth
+  #- lighthouse-nethermind,nimbus-erigon,teku-geth
+  #- lodestar-erigon,teku-nethermind,grandine-reth
+  #- grandine-geth,teku-erigon,lodestar-nethermind
+  #- teku-reth,grandine-erigon,nimbus-geth
+  assertoorTests:
+  - assertoor-tests/block-proposal-check.yaml