From b904e126852a68918ae15e0d0843b7d68070ca95 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 00:59:25 +0000 Subject: [PATCH 01/19] chore(NA): simple changes on bazelrc --- .bazelrc.common | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.bazelrc.common b/.bazelrc.common index a53d1b80724832..6cf8114dfb5a75 100644 --- a/.bazelrc.common +++ b/.bazelrc.common @@ -5,18 +5,22 @@ # # The full list of Bazel options: https://docs.bazel.build/versions/master/command-line-reference.html -# Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches) -build --disk_cache=bazel-cache/disk-cache +# Local Cache Settings +## Avoid cache results from being corrupt when changing source during build +common --experimental_guard_against_concurrent_changes -# Bazel repo cache settings -build --repository_cache=bazel-cache/repository-cache +## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches) +build --disk_cache=~/.bazel-cache/disk-cache + +## Bazel repo cache settings +build --repository_cache=~/.bazel-cache/repository-cache # Bazel will create symlinks from the workspace directory to output artifacts. -# Build results will be placed in a directory called "bazel-dist/bin" +# Build results will be placed in a directory called "bazel/bin" # This will still create a bazel-out symlink in # the project directory, which must be excluded from the # editor's search path. -build --symlink_prefix=bazel-dist/ +build --symlink_prefix=bazel/ # To disable the symlinks altogether (including bazel-out) we can use # build --symlink_prefix=/ # however this makes it harder to find outputs. @@ -37,9 +41,7 @@ common --color=yes build --show_task_finish build --noshow_progress build --noshow_loading_progress - -## enforced default values -build --show_result=1 +build --show_result=0 # Specifies desired output mode for running tests. # Valid values are @@ -78,7 +80,8 @@ test:debug --test_output=streamed --test_strategy=exclusive --test_timeout=9999 # The node process will break before user code starts and wait for the debugger to connect. run:debug --define=VERBOSE_LOGS=1 -- --node_options=--inspect-brk # The following option will change the build output of certain rules such as terser and may not be desirable in all cases -build:debug --compilation_mode=dbg +# It will also output both the repo cache and action cache to a folder inside the repo +build:debug --compilation_mode=dbg --show_result=1 --disk_cache=bazel-cache/disk-cache --repository_cache=bazel-cache/repository-cache # Turn off legacy external runfiles # This prevents accidentally depending on this feature, which Bazel will remove. From efb6a45a86fdd9306fdee8c6410d57b2aef42c11 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 03:11:24 +0000 Subject: [PATCH 02/19] chore(NA): integrate bazel tools with BuildBuddy and remote cache service --- .bazelrc | 1 + .gitignore | 2 +- src/dev/ci_setup/.bazelrc-ci | 4 ++++ src/dev/ci_setup/load_env_keys.sh | 3 +++ src/dev/ci_setup/setup.sh | 10 ++++++++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index fd469d1203a821..7439d33ee00a68 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,6 +2,7 @@ # Import shared settings first so we can override below import %workspace%/.bazelrc.common +## Disabled for now # Remote cache settings for local env # build --remote_cache=https://storage.googleapis.com/kibana-bazel-cache # build --incompatible_remote_results_ignore_disk=true diff --git a/.gitignore b/.gitignore index 2d7dd52e3ef9e2..c7d7e37732ca04 100644 --- a/.gitignore +++ b/.gitignore @@ -78,5 +78,5 @@ report.asciidoc .yarn-local-mirror # Bazel -/bazel-* +/bazel /.bazelrc.user diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index 5b345d3c9e2074..c2daf0abdd10aa 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -5,6 +5,10 @@ # Import and load bazelrc common settings for ci env try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common +# Import and load bazelrc ci auth settings for BuildBuddy +try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.auth + +## Disabled for now as we are using BuildBuddy # Remote cache settings for ci env # build --google_default_credentials # build --remote_upload_local_results=true diff --git a/src/dev/ci_setup/load_env_keys.sh b/src/dev/ci_setup/load_env_keys.sh index 62d29db232eae9..5f7a6c26bab21b 100644 --- a/src/dev/ci_setup/load_env_keys.sh +++ b/src/dev/ci_setup/load_env_keys.sh @@ -34,6 +34,9 @@ else PERCY_TOKEN=$(retry 5 vault read -field=value secret/kibana-issues/dev/percy) export PERCY_TOKEN + KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) + export KIBANA_BUILDBUDDY_CI_API_KEY + # remove vault related secrets unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR fi diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index e5e21e312b0ddb..be5249f0a16f7d 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -70,3 +70,13 @@ fi ### copy .bazelrc-ci into $HOME/.bazelrc ### cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; + +### +### generate .bazelrc-ci.auth +### +echo "# Generated by src/dev/ci_setup/setup.sh" > src/dev/ci_setup/.bazelrc-ci.auth +echo "build --bes_results_url=https://app.buildbuddy.io/invocation/" >> src/dev/ci_setup/.bazelrc-ci.auth +echo "build --bes_backend=grpcs://cloud.buildbuddy.io" >> src/dev/ci_setup/.bazelrc-ci.auth +echo "build --remote_cache=grpcs://cloud.buildbuddy.io" >> src/dev/ci_setup/.bazelrc-ci.auth +echo "build --remote_timeout=3600" >> src/dev/ci_setup/.bazelrc-ci.auth +echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> src/dev/ci_setup/.bazelrc-ci.auth From 6cc51bdd56fb3cbf3ef6e9e6078542807965ea10 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 13:09:01 +0000 Subject: [PATCH 03/19] chore(NA) fix bazelrc line config --- .bazelrc.common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc.common b/.bazelrc.common index 6cf8114dfb5a75..491b2eb8ef8216 100644 --- a/.bazelrc.common +++ b/.bazelrc.common @@ -81,7 +81,7 @@ test:debug --test_output=streamed --test_strategy=exclusive --test_timeout=9999 run:debug --define=VERBOSE_LOGS=1 -- --node_options=--inspect-brk # The following option will change the build output of certain rules such as terser and may not be desirable in all cases # It will also output both the repo cache and action cache to a folder inside the repo -build:debug --compilation_mode=dbg --show_result=1 --disk_cache=bazel-cache/disk-cache --repository_cache=bazel-cache/repository-cache +build:debug --compilation_mode=dbg --show_result=1 --disk_cache=bazel/disk-cache --repository_cache=bazel/repository-cache # Turn off legacy external runfiles # This prevents accidentally depending on this feature, which Bazel will remove. From 51bd5103bf6e870031fad3396fdc99430e1906cd Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 13:15:43 +0000 Subject: [PATCH 04/19] chore(NA): move non auth settings out of bazelrc.auth --- src/dev/ci_setup/.bazelrc-ci | 6 ++++++ src/dev/ci_setup/setup.sh | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index c2daf0abdd10aa..ddfdbc60e2c6dc 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -8,6 +8,12 @@ try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common # Import and load bazelrc ci auth settings for BuildBuddy try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.auth +# BuildBuddy Remote Settings +build --bes_results_url=https://app.buildbuddy.io/invocation/ +build --bes_backend=grpcs://cloud.buildbuddy.io +build --remote_cache=grpcs://cloud.buildbuddy.io +build --remote_timeout=3600 + ## Disabled for now as we are using BuildBuddy # Remote cache settings for ci env # build --google_default_credentials diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index be5249f0a16f7d..c4951adc72aa1a 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -75,8 +75,4 @@ cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; ### generate .bazelrc-ci.auth ### echo "# Generated by src/dev/ci_setup/setup.sh" > src/dev/ci_setup/.bazelrc-ci.auth -echo "build --bes_results_url=https://app.buildbuddy.io/invocation/" >> src/dev/ci_setup/.bazelrc-ci.auth -echo "build --bes_backend=grpcs://cloud.buildbuddy.io" >> src/dev/ci_setup/.bazelrc-ci.auth -echo "build --remote_cache=grpcs://cloud.buildbuddy.io" >> src/dev/ci_setup/.bazelrc-ci.auth -echo "build --remote_timeout=3600" >> src/dev/ci_setup/.bazelrc-ci.auth echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> src/dev/ci_setup/.bazelrc-ci.auth From 64a800ad8c9d4e4357b7e4e97370eb64741e4b2a Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 14:04:22 +0000 Subject: [PATCH 05/19] chore(NA): output home dir --- src/dev/ci_setup/setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index c4951adc72aa1a..abf73ee83c6306 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -76,3 +76,6 @@ cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; ### echo "# Generated by src/dev/ci_setup/setup.sh" > src/dev/ci_setup/.bazelrc-ci.auth echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> src/dev/ci_setup/.bazelrc-ci.auth + +echo "$HOME" +exit 1 From d35c51bca910bb0ad0140a2df422cbdc94fda161 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 14:22:37 +0000 Subject: [PATCH 06/19] chore(NA): load .bazelrc-ci.auth from /Users/tiagocosta dir --- src/dev/ci_setup/.bazelrc-ci | 2 +- src/dev/ci_setup/setup.sh | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index ddfdbc60e2c6dc..8650a96ceaaa40 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -6,7 +6,7 @@ try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common # Import and load bazelrc ci auth settings for BuildBuddy -try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.auth +try-import /var/lib/jenkins/.bazelrc-ci.auth # BuildBuddy Remote Settings build --bes_results_url=https://app.buildbuddy.io/invocation/ diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index abf73ee83c6306..6354144e364586 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -10,6 +10,17 @@ echo " -- PARENT_DIR='$PARENT_DIR'" echo " -- KIBANA_PKG_BRANCH='$KIBANA_PKG_BRANCH'" echo " -- TEST_ES_SNAPSHOT_VERSION='$TEST_ES_SNAPSHOT_VERSION'" +### +### copy .bazelrc-ci into $HOME/.bazelrc +### +cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; + +### +### generate .bazelrc-ci.auth into "$HOME/.bazelrc-ci.auth"; +### +echo "# Generated by src/dev/ci_setup/setup.sh" > "$HOME/.bazelrc-ci.auth" +echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc-ci.auth" + ### ### install dependencies ### @@ -66,16 +77,3 @@ if [ "$GIT_CHANGES" ]; then exit 1 fi -### -### copy .bazelrc-ci into $HOME/.bazelrc -### -cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; - -### -### generate .bazelrc-ci.auth -### -echo "# Generated by src/dev/ci_setup/setup.sh" > src/dev/ci_setup/.bazelrc-ci.auth -echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> src/dev/ci_setup/.bazelrc-ci.auth - -echo "$HOME" -exit 1 From 2212d31223421cb587a1a8a397361c0dba2e91e5 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 3 Feb 2021 16:22:28 +0000 Subject: [PATCH 07/19] chore(NA): remove bazelrc auth file and append directly into home bazelrc --- src/dev/ci_setup/.bazelrc-ci | 10 +--------- src/dev/ci_setup/setup.sh | 6 +++--- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index 8650a96ceaaa40..e325782af5bda5 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -5,16 +5,8 @@ # Import and load bazelrc common settings for ci env try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common -# Import and load bazelrc ci auth settings for BuildBuddy -try-import /var/lib/jenkins/.bazelrc-ci.auth - -# BuildBuddy Remote Settings +# BuildBuddy remote settings including cache build --bes_results_url=https://app.buildbuddy.io/invocation/ build --bes_backend=grpcs://cloud.buildbuddy.io build --remote_cache=grpcs://cloud.buildbuddy.io build --remote_timeout=3600 - -## Disabled for now as we are using BuildBuddy -# Remote cache settings for ci env -# build --google_default_credentials -# build --remote_upload_local_results=true diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index 6354144e364586..50acdc4b24d07d 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -16,10 +16,10 @@ echo " -- TEST_ES_SNAPSHOT_VERSION='$TEST_ES_SNAPSHOT_VERSION'" cp "src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; ### -### generate .bazelrc-ci.auth into "$HOME/.bazelrc-ci.auth"; +### append auth token to buildbuddy into "$HOME/.bazelrc"; ### -echo "# Generated by src/dev/ci_setup/setup.sh" > "$HOME/.bazelrc-ci.auth" -echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc-ci.auth" +echo "# Appended by src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc" +echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" ### ### install dependencies From 347fc132399b8cf6dacd94736c17101e5eb8d441 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 04:56:35 +0000 Subject: [PATCH 08/19] chore(NA): comment announce option --- src/dev/ci_setup/.bazelrc-ci.common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev/ci_setup/.bazelrc-ci.common b/src/dev/ci_setup/.bazelrc-ci.common index 3f58e4e03a1785..2ea589ab22624d 100644 --- a/src/dev/ci_setup/.bazelrc-ci.common +++ b/src/dev/ci_setup/.bazelrc-ci.common @@ -5,7 +5,7 @@ build --noshow_progress # Print all the options that apply to the build. -build --announce_rc +# build --announce_rc # More details on failures build --verbose_failures=true From 56ad7c5dece8d5365ad1e35ce39222311a33b636 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:01:33 +0000 Subject: [PATCH 09/19] chore(NA): integrate build buddy metadata --- .bazelrc | 7 ++++ src/dev/bazel_workspace_status.sh | 54 +++++++++++++++++++++++++++++++ src/dev/ci_setup/.bazelrc-ci | 7 +++- 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 src/dev/bazel_workspace_status.sh diff --git a/.bazelrc b/.bazelrc index ce752b73befb48..9fd070b714977a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -8,3 +8,10 @@ import %workspace%/.bazelrc.common # build --incompatible_remote_results_ignore_disk=true # build --remote_accept_cached=true # build --remote_upload_local_results=false + +# BuildBuddy +## Metadata settings +# Enable this in case you want to share your build info +# build --build_metadata=VISIBILITY=PUBLIC +build --build_metadata=TEST_GROUPS=//packages + diff --git a/src/dev/bazel_workspace_status.sh b/src/dev/bazel_workspace_status.sh new file mode 100644 index 00000000000000..a2063bc81756e9 --- /dev/null +++ b/src/dev/bazel_workspace_status.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Inspired on https://github.com/buildbuddy-io/buildbuddy/blob/master/workspace_status.sh +# This script will be run bazel when building process starts to +# generate key-value information that represents the status of the +# workspace. The output should be like +# +# KEY1 VALUE1 +# KEY2 VALUE2 +# +# If the script exits with non-zero code, it's considered as a failure +# and the output will be discarded. + +# Git repo +repo_url=$(git config --get remote.origin.url) +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "REPO_URL ${repo_url}" + +# Commit SHA +commit_sha=$(git rev-parse HEAD) +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "COMMIT_SHA ${commit_sha}" + +# Git branch +repo_url=$(git rev-parse --abbrev-ref HEAD) +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "GIT_BRANCH ${repo_url}" + +# Tree status +git diff-index --quiet HEAD -- +if [[ $? == 0 ]]; +then + tree_status="Clean" +else + tree_status="Modified" +fi +echo "GIT_TREE_STATUS ${tree_status}" + +# Host +host=$(hostname | sed 's|\(.*\)-.*|\1|') +if [[ $? != 0 ]]; +then + exit 1 +fi +echo "HOST ${host}" diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index e325782af5bda5..71bb17157b5032 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -5,8 +5,13 @@ # Import and load bazelrc common settings for ci env try-import %workspace%/src/dev/ci_setup/.bazelrc-ci.common -# BuildBuddy remote settings including cache +# BuildBuddy settings +## Remote settings including cache build --bes_results_url=https://app.buildbuddy.io/invocation/ build --bes_backend=grpcs://cloud.buildbuddy.io build --remote_cache=grpcs://cloud.buildbuddy.io build --remote_timeout=3600 + +## Metadata settings +build --workspace_status_command=$(pwd)/src/dev/workspace_status.sh +build --build_metadata=ROLE=CI From 1dc7faf7677bdd8bd7709f5345c3081469107e2a Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:06:43 +0000 Subject: [PATCH 10/19] chore(NA): update src/dev/ci_setup/.bazelrc-ci Co-authored-by: Tyler Smalley --- src/dev/ci_setup/.bazelrc-ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index 71bb17157b5032..a6399d2d28dbe8 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -13,5 +13,5 @@ build --remote_cache=grpcs://cloud.buildbuddy.io build --remote_timeout=3600 ## Metadata settings -build --workspace_status_command=$(pwd)/src/dev/workspace_status.sh +build --workspace_status_command=$(pwd)/src/dev/bazel_workspace_status.sh build --build_metadata=ROLE=CI From 12e7c10b6ed801f2342d4f11de52c827e84b01dd Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:10:29 +0000 Subject: [PATCH 11/19] chore(NA): move build metadata integation to common confdig --- .bazelrc | 1 + src/dev/ci_setup/.bazelrc-ci | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 9fd070b714977a..b28f955786e623 100644 --- a/.bazelrc +++ b/.bazelrc @@ -11,6 +11,7 @@ import %workspace%/.bazelrc.common # BuildBuddy ## Metadata settings +build --workspace_status_command=$(pwd)/src/dev/workspace_status.sh # Enable this in case you want to share your build info # build --build_metadata=VISIBILITY=PUBLIC build --build_metadata=TEST_GROUPS=//packages diff --git a/src/dev/ci_setup/.bazelrc-ci b/src/dev/ci_setup/.bazelrc-ci index 71bb17157b5032..ef6fab3a30590a 100644 --- a/src/dev/ci_setup/.bazelrc-ci +++ b/src/dev/ci_setup/.bazelrc-ci @@ -13,5 +13,4 @@ build --remote_cache=grpcs://cloud.buildbuddy.io build --remote_timeout=3600 ## Metadata settings -build --workspace_status_command=$(pwd)/src/dev/workspace_status.sh build --build_metadata=ROLE=CI From 5a280f092b923ef9e7008b41a5c35a2de7570a46 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:15:09 +0000 Subject: [PATCH 12/19] chore(NA): fix problem on bazel file location --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index b28f955786e623..158338ec5f093c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -11,7 +11,7 @@ import %workspace%/.bazelrc.common # BuildBuddy ## Metadata settings -build --workspace_status_command=$(pwd)/src/dev/workspace_status.sh +build --workspace_status_command=$(pwd)/src/dev/bazel_workspace_status.sh # Enable this in case you want to share your build info # build --build_metadata=VISIBILITY=PUBLIC build --build_metadata=TEST_GROUPS=//packages From 71a92148ab1dbba7ca7d342a9506ba1ee682b5ca Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:16:55 +0000 Subject: [PATCH 13/19] chore(NA): correct sh file permissions --- src/dev/bazel_workspace_status.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/dev/bazel_workspace_status.sh diff --git a/src/dev/bazel_workspace_status.sh b/src/dev/bazel_workspace_status.sh old mode 100644 new mode 100755 From f831fbec8693fb18df49a39f48738c818d27d8ae Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:24:48 +0000 Subject: [PATCH 14/19] chore(NA): only get host on CI --- src/dev/bazel_workspace_status.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/dev/bazel_workspace_status.sh b/src/dev/bazel_workspace_status.sh index a2063bc81756e9..162c607c8eacfb 100755 --- a/src/dev/bazel_workspace_status.sh +++ b/src/dev/bazel_workspace_status.sh @@ -46,9 +46,11 @@ fi echo "GIT_TREE_STATUS ${tree_status}" # Host -host=$(hostname | sed 's|\(.*\)-.*|\1|') -if [[ $? != 0 ]]; -then - exit 1 +if [ "$CI" = "true" ]; then + host=$(hostname | sed 's|\(.*\)-.*|\1|') + if [[ $? != 0 ]]; + then + exit 1 + fi + echo "HOST ${host}" fi -echo "HOST ${host}" From 889340de0b9eeb862985aafc8760f7a5df5873e3 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 19:25:44 +0000 Subject: [PATCH 15/19] chore(NA): add cores into host info on CI --- src/dev/bazel_workspace_status.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dev/bazel_workspace_status.sh b/src/dev/bazel_workspace_status.sh index 162c607c8eacfb..efaca4bb98849b 100755 --- a/src/dev/bazel_workspace_status.sh +++ b/src/dev/bazel_workspace_status.sh @@ -48,9 +48,10 @@ echo "GIT_TREE_STATUS ${tree_status}" # Host if [ "$CI" = "true" ]; then host=$(hostname | sed 's|\(.*\)-.*|\1|') + cores=$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}' ) if [[ $? != 0 ]]; then exit 1 fi - echo "HOST ${host}" + echo "HOST ${host}-${cores}" fi From 2c467cd3211dc714d061d8a84c6afceb6b0b841f Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 21:00:33 +0000 Subject: [PATCH 16/19] chore(NA): sync with last settings to setup bazelisk tools on ci --- src/dev/ci_setup/setup_env.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index 5dac270239c4af..dbe44e26c782fd 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -175,4 +175,21 @@ if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA fi +### +### copy .bazelrc-ci into $HOME/.bazelrc +### +cp -f "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; + +### +### append auth token to buildbuddy into "$HOME/.bazelrc"; +### +echo "# Appended by $KIBANA_DIR/src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc" +echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" + +### +### make sure bazelisk is installed on CI +### +bazeliskVersion=$(head -n 1 "$KIBANA_DIR/.bazeliskversion") +yarn global add "@bazel/bazelisk@${bazeliskVersion}" + export CI_ENV_SETUP=true From 4267ea0cf13bbb8571cd44b5197e0acb853fb5a2 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 4 Feb 2021 22:28:05 +0000 Subject: [PATCH 17/19] chore(NA): sync last changes on ci setup env --- src/dev/ci_setup/.bazelrc-ci.common | 3 --- src/dev/ci_setup/setup_env.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/dev/ci_setup/.bazelrc-ci.common b/src/dev/ci_setup/.bazelrc-ci.common index 2ea589ab22624d..9d00ee5639741f 100644 --- a/src/dev/ci_setup/.bazelrc-ci.common +++ b/src/dev/ci_setup/.bazelrc-ci.common @@ -4,8 +4,5 @@ # Don't be spammy in the logs build --noshow_progress -# Print all the options that apply to the build. -# build --announce_rc - # More details on failures build --verbose_failures=true diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index dbe44e26c782fd..e35f4dd9c1ad6c 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -190,6 +190,6 @@ echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" ### make sure bazelisk is installed on CI ### bazeliskVersion=$(head -n 1 "$KIBANA_DIR/.bazeliskversion") -yarn global add "@bazel/bazelisk@${bazeliskVersion}" +npm install -g "@bazel/bazelisk@${bazeliskVersion}" export CI_ENV_SETUP=true From cc503f8fcf84fdbf0bc3e07733ae751b3fb68dba Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 5 Feb 2021 02:37:12 +0000 Subject: [PATCH 18/19] chore(NA): sync settings on ci setup with the other PR --- src/dev/ci_setup/setup_env.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index e35f4dd9c1ad6c..06304e6ae41858 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -119,6 +119,10 @@ yarn config set yarn-offline-mirror "$cacheDir/yarn-offline-cache" ### yarnGlobalDir="$(yarn global bin)" export PATH="$PATH:$yarnGlobalDir" +# In some situations looks lik the yarn binaries added through +# yarn global add will not be placed on yarn global bin +# but on $HOME/.yarn/bin. I'm also adding it into the PATH +export PATH="$PATH:$HOME/.yarn/bin" # use a proxy to fetch chromedriver/geckodriver asset export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" @@ -186,10 +190,4 @@ cp -f "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"; echo "# Appended by $KIBANA_DIR/src/dev/ci_setup/setup.sh" >> "$HOME/.bazelrc" echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" -### -### make sure bazelisk is installed on CI -### -bazeliskVersion=$(head -n 1 "$KIBANA_DIR/.bazeliskversion") -npm install -g "@bazel/bazelisk@${bazeliskVersion}" - export CI_ENV_SETUP=true From 857058a00f4350b6bdb0a04fe1cc3f493add879a Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 5 Feb 2021 21:59:00 +0000 Subject: [PATCH 19/19] chore(NA): remove yarn export --- src/dev/ci_setup/setup_env.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index 06304e6ae41858..0b835d4b9fa947 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -119,10 +119,6 @@ yarn config set yarn-offline-mirror "$cacheDir/yarn-offline-cache" ### yarnGlobalDir="$(yarn global bin)" export PATH="$PATH:$yarnGlobalDir" -# In some situations looks lik the yarn binaries added through -# yarn global add will not be placed on yarn global bin -# but on $HOME/.yarn/bin. I'm also adding it into the PATH -export PATH="$PATH:$HOME/.yarn/bin" # use a proxy to fetch chromedriver/geckodriver asset export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"