Migrated CI from CircleCI to GitHub Actions (#3)

acelyc111 · Aug 1, 2023 · 6249a7d · 6249a7d
1 parent 3f7c92b
commit 6249a7d
Show file tree

Hide file tree

Showing 28 changed files with 843 additions and 3 deletions.
diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml
@@ -0,0 +1,7 @@
+name: build-folly
+runs:
+  using: composite
+  steps:
+  - name: Build folly and dependencies
+    run: make build_folly -j4
+    shell: bash
diff --git a/.github/actions/build-for-benchmarks/action.yml b/.github/actions/build-for-benchmarks/action.yml
@@ -0,0 +1,8 @@
+name: build-for-benchmarks
+runs:
+  using: composite
+  steps:
+  - uses: "./.github/actions/pre-steps"
+  - name: Linux build for benchmarks
+    run: make V=1 J=4 -j4 release
+    shell: bash
diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml
@@ -0,0 +1,10 @@
+name: increase-max-open-files-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Increase max open files
+    run: |-
+      sudo sysctl -w kern.maxfiles=1048576
+      sudo sysctl -w kern.maxfilesperproc=1048576
+      sudo launchctl limit maxfiles 1048576
+    shell: bash
diff --git a/.github/actions/install-cmake-on-macos/action.yml b/.github/actions/install-cmake-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-cmake-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install cmake on macos
+    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
+    shell: bash
diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-gflags-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install gflags on macos
+    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
+    shell: bash
diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml
@@ -0,0 +1,7 @@
+name: install-gflags
+runs:
+  using: composite
+  steps:
+  - name: Install gflags
+    run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+    shell: bash
diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml
@@ -0,0 +1,7 @@
+name: install-jdk8-on-macos
+runs:
+  using: composite
+  steps:
+  - name: Install JDK 8 on macos
+    run: brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
+    shell: bash
diff --git a/.github/actions/perform-benchmarks/action.yml b/.github/actions/perform-benchmarks/action.yml
@@ -0,0 +1,22 @@
+name: perform-benchmarks
+runs:
+  using: composite
+  steps:
+  - name: Test low-variance benchmarks
+    run: "./tools/benchmark_ci.py --db_dir /tmp/rocksdb-benchmark-datadir --output_dir /tmp/benchmark-results --num_keys 20000000"
+    env:
+      LD_LIBRARY_PATH: "/usr/local/lib"
+      DURATION_RO: 300
+      DURATION_RW: 500
+      NUM_THREADS: 1
+      MAX_BACKGROUND_JOBS: 4
+      CI_TESTS_ONLY: 'true'
+      WRITE_BUFFER_SIZE_MB: 16
+      TARGET_FILE_SIZE_BASE_MB: 16
+      MAX_BYTES_FOR_LEVEL_BASE_MB: 64
+      COMPRESSION_TYPE: none
+      CACHE_INDEX_AND_FILTER_BLOCKS: 1
+      MIN_LEVEL_TO_COMPRESS: 3
+      CACHE_SIZE_MB: 10240
+      MB_WRITE_PER_SEC: 2
+    shell: bash
diff --git a/.github/actions/post-benchmarks/action.yml b/.github/actions/post-benchmarks/action.yml
@@ -0,0 +1,15 @@
+name: post-benchmarks
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v3.1.1
+    with:
+      path: "/tmp/benchmark-results"
+  - name: Send benchmark report to visualisation
+    run: echo "skip this job, we don't have a visualisation yet"
+#    run: |-
+#      set +e
+#      set +o pipefail
+#      ./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
+#      true
+#    shell: bash
diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml
@@ -0,0 +1,25 @@
+name: post-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v3.1.1
+    with:
+      path: "/tmp/test-results"
+  - uses: actions/upload-artifact@v3.1.1
+    with:
+      path: LOG
+  - name: Compress Test Logs
+    run: tar -cvzf t.tar.gz t
+    if: failure()
+    shell: bash
+  - uses: actions/upload-artifact@v3.1.1
+    with:
+      path: t.tar.gz
+  - run: |-
+      mkdir -p /tmp/core_dumps
+      cp core.* /tmp/core_dumps
+    if: failure()
+    shell: bash
+  - uses: actions/upload-artifact@v3.1.1
+    with:
+      path: "/tmp/core_dumps"
diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml
@@ -0,0 +1,20 @@
+name: pre-steps-macos
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v3.5.0
+  - name: Setup Environment Variables
+    run: |-
+      echo "CIRCLECI=1" >> $GITHUB_ENV
+      echo "GTEST_THROW_ON_FAILURE=0" >> $GITHUB_ENV
+      echo "GTEST_OUTPUT=xml:/tmp/test-results/" >> $GITHUB_ENV
+      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> $GITHUB_ENV
+      echo "GTEST_COLOR=1" >> $GITHUB_ENV
+      echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
+      echo "CTEST_TEST_TIMEOUT=300" >> $GITHUB_ENV
+      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $GITHUB_ENV
+      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $GITHUB_ENV
+      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $GITHUB_ENV
+      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $GITHUB_ENV
+      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $GITHUB_ENV
+    shell: bash
diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml
@@ -0,0 +1,27 @@
+name: pre-steps
+runs:
+  using: composite
+  steps:
+  - name: Free Disk Space
+    run: |-
+      rm -rf /usr/local/lib/android
+      rm -rf /usr/share/dotnet
+      rm -rf /opt/ghc
+      apt-get remove -y 'php.*'
+    shell: bash
+  - uses: actions/checkout@v3.5.0
+  - name: Setup Environment Variables
+    run: |-
+      echo "CIRCLECI=1" >> $GITHUB_ENV
+      echo "GTEST_THROW_ON_FAILURE=0" >> $GITHUB_ENV
+      echo "GTEST_OUTPUT=xml:/tmp/test-results/" >> $GITHUB_ENV
+      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> $GITHUB_ENV
+      echo "GTEST_COLOR=1" >> $GITHUB_ENV
+      echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
+      echo "CTEST_TEST_TIMEOUT=300" >> $GITHUB_ENV
+      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $GITHUB_ENV
+      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $GITHUB_ENV
+      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $GITHUB_ENV
+      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $GITHUB_ENV
+      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $GITHUB_ENV
+    shell: bash
diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml
@@ -0,0 +1,7 @@
+name: setup-folly
+runs:
+  using: composite
+  steps:
+  - name: Checkout folly sources
+    run: make checkout_folly -j4
+    shell: bash
diff --git a/.github/actions/windows-build-steps/action.yml b/.github/actions/windows-build-steps/action.yml
@@ -0,0 +1,30 @@
+name: windows-build-steps
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v3.5.0
+  - name: Install thirdparty dependencies
+    run: |-
+      mkdir $Env:THIRDPARTY_HOME
+      cd $Env:THIRDPARTY_HOME
+      echo "Building Snappy dependency..."
+      curl https://github.com/google/snappy/archive/refs/tags/1.1.8.zip -O snappy-1.1.8.zip
+      unzip -q snappy-1.1.8.zip
+      cd snappy-1.1.8
+      mkdir build
+      cd build
+      & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -j4 ..
+      msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+    shell: bash
+  - name: Build RocksDB
+    run: |-
+      mkdir build
+      cd build
+      & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 -j4 ..
+      cd ..
+      echo "Building with VS version: $Env:CMAKE_GENERATOR"
+      msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+    shell: bash
+  - name: Test RocksDB
+    run: build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 4
+    shell: powershell.exe
diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml
@@ -0,0 +1,13 @@
+name: benchmark-linux
+on:
+  schedule:
+  - cron: 0 * * * *
+#   # 'filters' was not transformed because there is no suitable equivalent in GitHub Actions
+jobs:
+  benchmark-linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/build-for-benchmarks"
+    - uses: "./.github/actions/perform-benchmarks"
+    - uses: "./.github/actions/post-benchmarks"
diff --git a/.github/workflows/build-fuzzers.yml b/.github/workflows/build-fuzzers.yml
@@ -0,0 +1,15 @@
+name: build-fuzzers
+on: [push, pull_request]
+jobs:
+  build-fuzzers:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Build rocksdb lib
+      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+    - name: Build fuzzers
+      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
+    - uses: "./.github/actions/post-steps"
diff --git a/.github/workflows/jobs-java.yml b/.github/workflows/jobs-java.yml
@@ -0,0 +1,104 @@
+name: jobs-java
+on: [push, pull_request]
+jobs:
+  build-linux-java:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Test RocksDBJava
+      run: make V=1 J=4 -j4 jtest
+    - uses: "./.github/actions/post-steps"
+  build-linux-java-static:
+    runs-on: ubuntu-latest
+    container:
+      image: zjay437/rocksdb:0.6
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Build RocksDBJava Static Library
+      run: make V=1 J=4 -j4 rocksdbjavastatic
+    - uses: "./.github/actions/post-steps"
+  build-macos-java:
+    runs-on: macos-11
+    env:
+      JAVA_HOME: "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home"
+      ROCKSDB_DISABLE_JEMALLOC: 1
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: maxim-lobanov/setup-xcode@v1.5.1
+      with:
+        xcode-version: 12.5.1
+    - uses: "./.github/actions/increase-max-open-files-on-macos"
+    - uses: "./.github/actions/install-gflags-on-macos"
+    - uses: "./.github/actions/install-jdk8-on-macos"
+    - uses: "./.github/actions/pre-steps-macos"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Test RocksDBJava
+      run: make V=1 J=4 -j4 jtest
+    - uses: "./.github/actions/post-steps"
+  build-macos-java-static:
+    runs-on: macos-11
+    env:
+      JAVA_HOME: "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home"
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: maxim-lobanov/setup-xcode@v1.5.1
+      with:
+        xcode-version: 12.5.1
+    - uses: "./.github/actions/increase-max-open-files-on-macos"
+    - uses: "./.github/actions/install-gflags-on-macos"
+    - uses: "./.github/actions/install-cmake-on-macos"
+    - uses: "./.github/actions/install-jdk8-on-macos"
+    - uses: "./.github/actions/pre-steps-macos"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Build RocksDBJava x86 and ARM Static Libraries
+      run: make V=1 J=4 -j4 rocksdbjavastaticosx
+    - uses: "./.github/actions/post-steps"
+  build-macos-java-static-universal:
+    runs-on: macos-11
+    env:
+      JAVA_HOME: "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home"
+    steps:
+    - uses: actions/checkout@v3.5.0
+    - uses: maxim-lobanov/setup-xcode@v1.5.1
+      with:
+        xcode-version: 12.5.1
+    - uses: "./.github/actions/increase-max-open-files-on-macos"
+    - uses: "./.github/actions/install-gflags-on-macos"
+    - uses: "./.github/actions/install-cmake-on-macos"
+    - uses: "./.github/actions/install-jdk8-on-macos"
+    - uses: "./.github/actions/pre-steps-macos"
+    - name: Set Java Environment
+      run: |-
+        echo "JAVA_HOME=${JAVA_HOME}"
+        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_ENV
+        which java && java -version
+        which javac && javac -version
+    - name: Build RocksDBJava Universal Binary Static Library
+      run: make V=1 J=4 -j4 rocksdbjavastaticosx_ub
+    - uses: "./.github/actions/post-steps"
diff --git a/.github/workflows/jobs-linux-arm.yml b/.github/workflows/jobs-linux-arm.yml
@@ -0,0 +1,12 @@
+name: jobs-linux-arm
+on: [push, pull_request]
+jobs:
+  build-linux-arm:
+    runs-on: ubuntu-20.04
+    steps:
+    - run: echo "skip this job, Github Actions free plan does not support arm64 yet"
+#    - uses: actions/checkout@v3.5.0
+#    - uses: "./.github/actions/pre-steps"
+#    - uses: "./.github/actions/install-gflags"
+#    - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
+#    - uses: "./.github/actions/post-steps"