From ccdf9ac9853601a81ff21f8f42f5bd866dd4de75 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Fri, 22 Jul 2022 05:56:02 -0700 Subject: [PATCH] Reduce flakiness of CI (#34235) Summary: How the Hermes cache worked had a concurrency issue. It used to work by asking the Hermes repository for the latest commit and using that commit as cache key to try and retrieve a built version of Hermes to avoid to compile it more than once. The problem happened when the `build_hermes_macos` was building Hermes using a commit A. While building, another PR could be merged into `hermes:main`, creating commit B. When this happened, the tasks `test_ios` and `test_ios_rntester`would try to retrieve a cached version of Hermes using commit B. That version did not exist because Hermes was created using commit A, and the job would either fail or trying to build Hermes from source, ending up taking a lot of time. This PR attaches the `.hermes-cache-key-file` to the workspace and restores it in the other jobs, making sure that the same cache key is used in all three jobs. ## Changelog [General] [Changed] - Attach the `.hermes-cache-key-file` to the workspace to avoid race conditions for new PR landing on Hermes and changing the head commit between the time Hermes is built and the time it has to be consumed. Pull Request resolved: https://github.com/facebook/react-native/pull/34235 Test Plan: Tested manually, looking at the messages in CI. **build_hermes_macos** Screenshot 2022-07-21 at 15 40 02 **test_ios_rntester**: notice that the `echo` is not executed, meaning that the `if` does not evaluate to true and, therefore, the file has been correctly attached. Screenshot 2022-07-21 at 15 40 52 **test_ios**: notice that the `echo` is not executed, meaning that the `if` does not evaluate to true and, therefore, the file has been correctly attached. Screenshot 2022-07-21 at 15 46 33 Reviewed By: cortinico Differential Revision: D38037386 Pulled By: cipolleschi fbshipit-source-id: 4db4f7c478e1afb2e4a18ba3d3f70896ed41d235 --- .circleci/config.yml | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 107fba51965112..d4da82a4e9b16f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,9 +52,8 @@ references: gems_cache_key: &gems_cache_key v1-gems-{{ checksum "Gemfile.lock" }} gradle_cache_key: &gradle_cache_key v1-gradle-{{ checksum "gradle/wrapper/gradle-wrapper.properties" }}-{{ checksum "ReactAndroid/gradle.properties" }} hermes_cache_key: &hermes_cache_key v1-hermes-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/hermes/hermesversion" }} - hermes_sdk_cache_key: &hermes_sdk_cache_key v1-hermes-{{ checksum "sdks/.hermes-cache-key-file" }} hermes_windows_cache_key: &hermes_windows_cache_key v1-hermes-{{ .Environment.CIRCLE_JOB }}-{{ checksum "tmp/hermes/hermesversion" }} - hermes_tarball_cache_key: &hermes_tarball_cache_key v1-hermes-tarball-{{ checksum "sdks/.hermes-cache-key-file" }} + hermes_tarball_cache_key: &hermes_tarball_cache_key v1-hermes-tarball-{{ checksum "/tmp/hermes/cache/.hermes-cache-key-file" }} pods_cache_key: &pods_cache_key v6-pods-{{ .Environment.CIRCLE_JOB }}-{{ checksum "packages/rn-tester/Podfile.lock.bak" }}-{{ checksum "packages/rn-tester/Podfile" }} windows_yarn_cache_key: &windows_yarn_cache_key v1-win-yarn-cache-{{ arch }}-{{ checksum "yarn.lock" }} yarn_cache_key: &yarn_cache_key v5-yarn-cache-{{ .Environment.CIRCLE_JOB }} @@ -293,10 +292,13 @@ commands: - run: name: Setup Hermes cache command: | - HERMES_CACHE_KEY_FILE="sdks/.hermes-cache-key-file" + HERMES_CACHE_KEY_FILE="/tmp/hermes/cache/.hermes-cache-key-file" if [ ! -f "$HERMES_CACHE_KEY_FILE" ]; then + echo "File not found. Creating it using latest commit from main" + mkdir -p /tmp/hermes/cache git ls-remote https://github.com/facebook/hermes main | cut -f 1 > $HERMES_CACHE_KEY_FILE fi + cat $HERMES_CACHE_KEY_FILE - restore_cache: keys: - *hermes_tarball_cache_key @@ -309,13 +311,13 @@ commands: BASE_PATH=/tmp/hermes/hermes-runtime-darwin/ if [ ! -d $BASE_PATH ]; then echo "Hermes tarball base path not present ($BASE_PATH). Build it from source." - return + exit 0 fi TARBALL=$(ls /tmp/hermes/hermes-runtime-darwin/) TARBALL_PATH=$BASE_PATH$TARBALL if [ ! -f $TARBALL_PATH ]; then echo "Hermes tarball not present ($TARBALL_PATH). Build it from source." - return + exit 0 fi echo "export HERMES_ENGINE_TARBALL_PATH=$TARBALL_PATH" >> $BASH_ENV @@ -472,6 +474,8 @@ jobs: cd scripts sh run_ruby_tests.sh - run_yarn + - attach_workspace: + at: /tmp/hermes/ - run: | cd packages/rn-tester bundle check || bundle install @@ -761,6 +765,9 @@ jobs: - brew_install: package: cmake + - attach_workspace: + at: /tmp/hermes + - with_hermes_tarball_cache_span: set_tarball_path: True steps: @@ -1002,38 +1009,24 @@ jobs: brew install cmake - with_hermes_tarball_cache_span: steps: + - persist_to_workspace: + root: /tmp/hermes/ + paths: + - cache - run: name: Build the Hermes iOS frameworks command: | - if [[ -d "~/react-native/sdks/hermes/destroot" && -d "/tmp/hermes/hermes-runtime-darwin/" ]]; then - echo "destroot and tarball exists. Skip building" - return - fi - echo "either destroot or the tarball does not exists. Build from source" - cd ~/react-native/sdks/hermes ./utils/build-ios-framework.sh - run: name: Build the Hermes Mac frameworks command: | - if [[ -d "~/react-native/sdks/hermes/destroot" && -d "/tmp/hermes/hermes-runtime-darwin/" ]]; then - echo "destroot and tarball exists. Skip building" - return - fi - echo "either destroot or the tarball does not exists. Build from source" - cd ~/react-native/sdks/hermes ./utils/build-mac-framework.sh cp build_macosx/bin/hermesc /tmp/hermes/osx-bin/. - run: name: Package the Hermes Apple frameworks command: | - if [[ -d "~/react-native/sdks/hermes/destroot" && -d "/tmp/hermes/hermes-runtime-darwin/" ]]; then - echo "destroot and tarball exists. Skip building" - return - fi - echo "either destroot or the tarball does not exists. Build from source" - cd ~/react-native/sdks/hermes . ./utils/build-apple-framework.sh