From ed95267d98889b975cb8a22b072808e2c9e50648 Mon Sep 17 00:00:00 2001 From: Solonas Gousteris Date: Tue, 1 Mar 2022 20:50:52 +0200 Subject: [PATCH 1/2] bk: added pre-command hook The pre-command hook runs before the command steps. We need that to check for github labels, in order to run in parallel any steps we want --- .buildkite/hooks/pre-command | 39 ++++++++++++++++++++++++++++++++++++ .gitignore | 3 ++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100755 .buildkite/hooks/pre-command diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 000000000000..a6d130eba8e9 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,39 @@ +#!/bin/bash + +# This file should be kept in sync with vtools's pre-command file. +# https://github.com/redpanda-data/vtools/.buildkite/hooks/pre-command + +set -uo pipefail +set +e + +PARALLEL_STEPS=1 + +repo=$(echo $BUILDKITE_REPO | awk -F/ '{print $NF}' | awk -F. '{print $1}') + +if [[ ${BUILDKITE_PULL_REQUEST} != false ]]; then + labels=$(curl \ + --retry 3 --retry-connrefused --fail \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${GITHUB_API_TOKEN}" \ + "https://api.github.com/repos/redpanda-data/${repo}/issues/${BUILDKITE_PULL_REQUEST}/labels") + + if [[ $? == "0" ]]; then + set -e + parallel=$(echo "${labels}" | jq -r '.[].name|select(startswith("ci-repeat-"))|ltrimstr("ci-repeat-")' | head -1) + curl \ + -X "DELETE" \ + --retry 3 --retry-connrefused --silent \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${GITHUB_API_TOKEN}" \ + "https://api.github.com/repos/redpanda-data/${repo}/issues/${BUILDKITE_PULL_REQUEST}/labels/ci-repeat-$parallel" + + if ((parallel > 1 && parallel < 20)); then + PARALLEL_STEPS=$parallel + else + echo >&2 "Provided parallel number is out of range(2-19). Changing to 1." + PARALLEL_STEPS=1 + fi + fi +fi + +export PARALLEL_STEPS diff --git a/.gitignore b/.gitignore index 041311bb466a..516cc5f77232 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ *.out *.app /*build* +!.buildkite /dbuild bin _e2e_artifacts/ @@ -240,4 +241,4 @@ vbuild/ # VSCode java settings .classpath .factorypath -.settings/ \ No newline at end of file +.settings/ From 91eadd4cc9d39567e210cfceecc47be30c16c811 Mon Sep 17 00:00:00 2001 From: Solonas Gousteris Date: Fri, 4 Mar 2022 18:46:32 +0200 Subject: [PATCH 2/2] Added env var to skip the pre-command hook --- .buildkite/hooks/pre-command | 56 +++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index a6d130eba8e9..b991303db363 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,34 +6,38 @@ set -uo pipefail set +e -PARALLEL_STEPS=1 - -repo=$(echo $BUILDKITE_REPO | awk -F/ '{print $NF}' | awk -F. '{print $1}') - -if [[ ${BUILDKITE_PULL_REQUEST} != false ]]; then - labels=$(curl \ - --retry 3 --retry-connrefused --fail \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${GITHUB_API_TOKEN}" \ - "https://api.github.com/repos/redpanda-data/${repo}/issues/${BUILDKITE_PULL_REQUEST}/labels") - - if [[ $? == "0" ]]; then - set -e - parallel=$(echo "${labels}" | jq -r '.[].name|select(startswith("ci-repeat-"))|ltrimstr("ci-repeat-")' | head -1) - curl \ - -X "DELETE" \ - --retry 3 --retry-connrefused --silent \ +disable_hook="${DISABLE_PRE_COMMAND_HOOK:-0}" + +if [[ ${disable_hook} == 0 ]]; then + PARALLEL_STEPS=1 + + repo=$(echo $BUILDKITE_REPO | awk -F/ '{print $NF}' | awk -F. '{print $1}') + + if [[ ${BUILDKITE_PULL_REQUEST} != false ]]; then + labels=$(curl \ + --retry 3 --retry-connrefused --fail \ -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: token ${GITHUB_API_TOKEN}" \ - "https://api.github.com/repos/redpanda-data/${repo}/issues/${BUILDKITE_PULL_REQUEST}/labels/ci-repeat-$parallel" - - if ((parallel > 1 && parallel < 20)); then - PARALLEL_STEPS=$parallel - else - echo >&2 "Provided parallel number is out of range(2-19). Changing to 1." - PARALLEL_STEPS=1 + "https://api.github.com/repos/redpanda-data/${repo}/issues/${BUILDKITE_PULL_REQUEST}/labels") + + if [[ $? == "0" ]]; then + set -e + parallel=$(echo "${labels}" | jq -r '.[].name|select(startswith("ci-repeat-"))|ltrimstr("ci-repeat-")' | head -1) + curl \ + -X "DELETE" \ + --retry 3 --retry-connrefused --silent \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${GITHUB_API_TOKEN}" \ + "https://api.github.com/repos/redpanda-data/${repo}/issues/${BUILDKITE_PULL_REQUEST}/labels/ci-repeat-$parallel" + + if ((parallel > 1 && parallel < 20)); then + PARALLEL_STEPS=$parallel + else + echo >&2 "Provided parallel number is out of range(2-19). Changing to 1." + PARALLEL_STEPS=1 + fi fi fi -fi -export PARALLEL_STEPS + export PARALLEL_STEPS +fi