From e7ec4b966e6942c328235a41b8cdd4e90adb4e05 Mon Sep 17 00:00:00 2001
From: Md Armughanuddin <52311490+armughan11@users.noreply.github.com>
Date: Mon, 17 Jul 2023 18:37:15 -0500
Subject: [PATCH] Added GCP JMH Benchmark Workflow  (#770)

Adds a workflow that is triggered via a "/benchmark" comment on a pull
request. Clones the main and PR branch on the GCP instance and runs the
`./gradlew jmh` on each one of them, returning the results as a comment
on the pull request.
---------

Co-authored-by: Manu Sridharan <msridhar@gmail.com>
---
 .github/workflows/gcloud_ssh.sh          | 11 ++++
 .github/workflows/get_repo_details.sh    | 20 +++++++
 .github/workflows/jmh-benchmark.yml      | 70 ++++++++++++++++++++++++
 .github/workflows/run_gcp_benchmarks.sh  | 20 +++++++
 .github/workflows/run_main_benchmarks.sh |  9 +++
 .github/workflows/run_pr_benchmarks.sh   |  9 +++
 6 files changed, 139 insertions(+)
 create mode 100644 .github/workflows/gcloud_ssh.sh
 create mode 100644 .github/workflows/get_repo_details.sh
 create mode 100644 .github/workflows/jmh-benchmark.yml
 create mode 100644 .github/workflows/run_gcp_benchmarks.sh
 create mode 100644 .github/workflows/run_main_benchmarks.sh
 create mode 100644 .github/workflows/run_pr_benchmarks.sh

diff --git a/.github/workflows/gcloud_ssh.sh b/.github/workflows/gcloud_ssh.sh
new file mode 100644
index 0000000000..80d9e68795
--- /dev/null
+++ b/.github/workflows/gcloud_ssh.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script is used to run commands on a Google Cloud instance via SSH
+
+# Define the variables for Google Cloud project, zone, username, and instance
+PROJECT_ID="ucr-ursa-major-sridharan-lab"
+ZONE="us-central1-a"
+USER="root"
+INSTANCE="nullway-jmh"
+
+gcloud compute ssh --project=$PROJECT_ID --zone=$ZONE $USER@$INSTANCE --command="$1"
diff --git a/.github/workflows/get_repo_details.sh b/.github/workflows/get_repo_details.sh
new file mode 100644
index 0000000000..55521665f4
--- /dev/null
+++ b/.github/workflows/get_repo_details.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script retrieves the repository and branch details of a GitHub pull request
+
+# Assign command line arguments to variables
+# GH_TOKEN is the GitHub authentication token
+# PR_NUMBER is the number of the pull request
+# REPO_NAME is the name of the repository
+GH_TOKEN="$1"
+PR_NUMBER="$2"
+REPO_NAME="$3"
+
+PR_DETAILS=$(curl -s -H "Authorization: token $GH_TOKEN" "https://api.github.com/repos/$REPO_NAME/pulls/$PR_NUMBER")
+
+REPO_FULL_NAME=$(echo "$PR_DETAILS" | jq -r .head.repo.full_name)
+BRANCH_NAME=$(echo "$PR_DETAILS" | jq -r .head.ref)
+
+# Export vars to GITHUB_ENV so they can be used by later scripts
+echo "REPO_FULL_NAME=$REPO_FULL_NAME" >> $GITHUB_ENV
+echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
diff --git a/.github/workflows/jmh-benchmark.yml b/.github/workflows/jmh-benchmark.yml
new file mode 100644
index 0000000000..858a411306
--- /dev/null
+++ b/.github/workflows/jmh-benchmark.yml
@@ -0,0 +1,70 @@
+# This GitHub Actions workflow runs JMH benchmarks when a new comment is created on a pull request
+name: Run JMH Benchmarks for Pull Request
+
+on:
+  issue_comment: # This workflow triggers when a comment is created
+    types: [created]
+
+jobs:
+  benchmarking:
+    # Only run this job if a comment on a pull request contains '/benchmark' and is a PR on the uber/NullAway repository
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark') && github.repository == 'uber/NullAway'
+    runs-on: ubuntu-latest 
+    permissions: write-all 
+
+    steps:
+    - name: Add reaction
+      uses: peter-evans/create-or-update-comment@v3
+      with:
+        comment-id: ${{ github.event.comment.id }}
+        reactions: '+1'
+        
+    - name: Checkout repository 
+      uses: actions/checkout@v3
+
+    - name: Set branch name 
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+      run: |
+        chmod +x ./.github/workflows/get_repo_details.sh 
+        ./.github/workflows/get_repo_details.sh "${{ secrets.GITHUB_TOKEN }}" "${{ github.event.issue.number }}" "${{ github.repository }}"
+
+    - id: 'auth'
+      name: Authenticating 
+      uses: 'google-github-actions/auth@v1' 
+      with:
+        credentials_json: '${{ secrets.GCP_SA_KEY_1 }}' 
+
+    - name: Set up Google Cloud SDK 
+      uses: google-github-actions/setup-gcloud@v1
+      
+    - name: Start VM
+      run: gcloud compute instances start nullway-jmh --zone=us-central1-a
+      
+    - name: Run benchmarks 
+      run: |
+        chmod +x ./.github/workflows/run_gcp_benchmarks.sh
+        ./.github/workflows/run_gcp_benchmarks.sh 
+        
+    - name: Cleanup
+      # Delete the branch directory on the Google Cloud instance
+      if: always()
+      run: |
+        ./.github/workflows/gcloud_ssh.sh " export BRANCH_NAME=${BRANCH_NAME} && rm -r -f $BRANCH_NAME"
+
+    - name: Formatting Benchmark # Create a text file containing the benchmark results
+      run:  |
+        (echo 'Main Branch:'; echo '```' ; cat main_text.txt; echo '```'; echo 'With This PR:'; echo '```' ; cat pr_text.txt; echo '```') > benchmark.txt
+   
+    - name: Comment Benchmark
+      uses: mshick/add-pr-comment@v2
+      if: always() # This step is for adding the comment
+      with:         
+        message-path: benchmark.txt  # The path to the message file to leave as a comment
+        message-id: benchmark
+    - name: Stop VM
+      if: always()
+      run: gcloud compute instances stop nullway-jmh --zone=us-central1-a
+
+      
+
diff --git a/.github/workflows/run_gcp_benchmarks.sh b/.github/workflows/run_gcp_benchmarks.sh
new file mode 100644
index 0000000000..059e72c4f6
--- /dev/null
+++ b/.github/workflows/run_gcp_benchmarks.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script is responsible for running benchmarks for a GitHub pull request and the main branch on Google Cloud Compute Engine (GCCE).
+
+
+chmod +x ./.github/workflows/gcloud_ssh.sh
+./.github/workflows/gcloud_ssh.sh "export BRANCH_NAME=${BRANCH_NAME} && mkdir $BRANCH_NAME"
+
+# Using gcloud compute scp to copy the bash scripts that will run the benchmarks onto the GCCE
+gcloud compute scp ./.github/workflows/run_pr_benchmarks.sh root@nullway-jmh:$BRANCH_NAME/ --zone=us-central1-a
+gcloud compute scp ./.github/workflows/run_main_benchmarks.sh root@nullway-jmh:$BRANCH_NAME/ --zone=us-central1-a
+
+# Running the benchmark script for the pull request branch and main branch on GCCE
+./.github/workflows/gcloud_ssh.sh " export BRANCH_NAME=${BRANCH_NAME} && export REPO_NAME=${REPO_FULL_NAME} && chmod +x $BRANCH_NAME/run_pr_benchmarks.sh && $BRANCH_NAME/run_pr_benchmarks.sh && cd && chmod +x $BRANCH_NAME/run_main_benchmarks.sh && $BRANCH_NAME/run_main_benchmarks.sh"
+
+# Copying the benchmark results from GCCE back to the Github runner for the PR branch
+gcloud compute scp root@nullway-jmh:$BRANCH_NAME/pr/NullAway/jmh/build/results/jmh/results.txt ./pr_text.txt --zone=us-central1-a
+
+# Copying the benchmark results from GCCE back to the Github runner for the main branch
+gcloud compute scp root@nullway-jmh:$BRANCH_NAME/main/NullAway/jmh/build/results/jmh/results.txt ./main_text.txt --zone=us-central1-a
diff --git a/.github/workflows/run_main_benchmarks.sh b/.github/workflows/run_main_benchmarks.sh
new file mode 100644
index 0000000000..c28e9231aa
--- /dev/null
+++ b/.github/workflows/run_main_benchmarks.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd $BRANCH_NAME/
+mkdir main
+cd main/
+git clone git@github.com:Uber/NullAway.git
+cd NullAway/
+
+./gradlew jmh
diff --git a/.github/workflows/run_pr_benchmarks.sh b/.github/workflows/run_pr_benchmarks.sh
new file mode 100644
index 0000000000..f711176899
--- /dev/null
+++ b/.github/workflows/run_pr_benchmarks.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd $BRANCH_NAME/
+mkdir pr
+cd pr/
+git clone --branch $BRANCH_NAME --single-branch git@github.com:$REPO_NAME.git NullAway
+cd NullAway/
+
+./gradlew jmh