ci: add native image builds for linux and macos (#690)

* ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases * fixup! ci: add native-image builds and releases
replikativ · Sep 9, 2024 · 7428d0a · 7428d0a
1 parent 099d8ba
commit 7428d0a
Show file tree

Hide file tree

Showing 10 changed files with 338 additions and 100 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,8 +1,10 @@
 version: 2.1
 
+setup: true
+
 orbs:
-  github-cli: circleci/github-cli@1.0
   tools: replikativ/clj-tools@0
+  continuation: circleci/continuation@0.1.2
 
 jobs:
   setup:
@@ -65,46 +67,20 @@ jobs:
           paths:
             - /home/circleci/.m2
   native-image:
-    machine:
-      image: default
-      resource_class: large
+    executor: tools/clojurecli
     steps:
       - attach_workspace:
           at: /home/circleci
       - run:
-          name: install graalvm
-          command: |
-            cd /home/circleci
-            wget https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.0/graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz
-            tar -xzf graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz
-            export PATH=/home/circleci/graalvm-community-openjdk-21+35.1/bin:$PATH
-            export JAVA_HOME=/home/circleci/graalvm-community-openjdk-21+35.1
-      - run:
-          name: install clojure
-          command: |
-            cd /home/circleci
-            curl -O https://download.clojure.org/install/linux-install-1.11.1.1165.sh
-            chmod +x linux-install-1.11.1.1165.sh
-            ./linux-install-1.11.1.1165.sh --prefix /home/circleci/clojure
-      - run:
-          name: install babashka
-          command: |
-            cd /home/circleci
-            curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
-            chmod +x install
-            ./install --dir /home/circleci/bin
-            export PATH=/home/circleci/bin:$PATH
-      - run:
-          name: build native-image
+          name: Generate config
           command: |
-            cd /home/circleci/replikativ
-            export PATH=/home/circleci/graalvm-community-openjdk-21+35.1/bin:/home/circleci/clojure/bin:$PATH
-            export JAVA_HOME=/home/circleci/graalvm-community-openjdk-21+35.1
-            bb ni-cli
-      - persist_to_workspace:
-          root: /home/circleci/
+            bb .circleci/scripts/gen_ci.clj > generated_config.yml
+      - continuation/continue:
+          configuration_path: generated_config.yml
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
           paths:
-            - replikativ/dthk
+            - /home/circleci/.m2
   persistent-set-test:
     executor: tools/clojurecli
     steps:
@@ -144,28 +120,6 @@ jobs:
           key: deps-{{ checksum "deps.edn" }}
           paths:
             - /home/circleci/.m2
-  native-image-test:
-    executor: tools/clojurecli
-    steps:
-      - attach_workspace:
-          at: /home/circleci
-      - run:
-          name: Run native-image test
-          command: |
-            cd /home/circleci/replikativ
-            bb test native-image
-          no_output_timeout: 5m
-  bb-pod-test:
-    executor: tools/clojurecli
-    steps:
-      - attach_workspace:
-          at: /home/circleci
-      - run:
-          name: Run bb-pod test
-          command: |
-            cd /home/circleci/replikativ
-            bb test bb-pod
-          no_output_timeout: 5m
   backward-compatibility-test:
     executor: tools/clojurecli
     steps:
@@ -218,20 +172,14 @@ jobs:
       - run:
           name: Deploy to clojars
           command: bb clojars
-  release:
+  release-jar:
     executor: tools/clojurecli
     steps:
       - attach_workspace:
           at: /home/circleci
       - run:
           name: Release jar on GitHub
           command: bb release jar
-      - run:
-          name: Release cli on GitHub
-          command: bb release native-cli
-          environment:
-            DTHK_PLATFORM: linux
-            DTHK_ARCH: x86_64
 
 workflows:
   build-test-and-deploy:
@@ -244,7 +192,9 @@ workflows:
           requires:
             - setup
       - native-image:
-          context: dockerhub-deploy
+          context:
+            - dockerhub-deploy
+            - github-token
           requires:
             - setup
       - format:
@@ -259,16 +209,6 @@ workflows:
           context: dockerhub-deploy
           requires:
             - build
-      - native-image-test:
-          context: dockerhub-deploy
-          requires:
-            - build
-            - native-image
-      - bb-pod-test:
-          context: dockerhub-deploy
-          requires:
-            - build
-            - native-image
       - backward-compatibility-test:
           context: dockerhub-deploy
           requires:
@@ -290,9 +230,8 @@ workflows:
             - persistent-set-test
             - hitchhiker-tree-test
             - backward-compatibility-test
-            - native-image-test
             - integration-test
-      - release:
+      - release-jar:
           context:
             - dockerhub-deploy
             - github-token
@@ -304,5 +243,4 @@ workflows:
             - persistent-set-test
             - hitchhiker-tree-test
             - backward-compatibility-test
-            - native-image-test
             - integration-test
diff --git a/.circleci/scripts/gen_ci.clj b/.circleci/scripts/gen_ci.clj
@@ -0,0 +1,176 @@
+(ns gen-ci
+  (:require
+   [babashka.tasks :as tasks]
+   [clj-yaml.core :as yaml]
+   [clojure.string :as str]
+   [flatland.ordered.map :refer [ordered-map]]))
+
+(def graalvm-version "22.0.2")
+
+(defn run
+  ([cmd-name cmd]
+   (run cmd-name cmd nil))
+  ([cmd-name cmd no-output-timeout]
+   (let [base {:run {:name    cmd-name
+                     :command cmd}}]
+     (if no-output-timeout
+       (assoc-in base [:run :no_output_timeout] no-output-timeout)
+       base))))
+
+(defn make-graalvm-url [arch]
+  (let [myarch (case arch
+                 "amd64" "x64"
+                 "aarch64" "aarch64")]
+    (str "https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-"
+         graalvm-version
+         "/graalvm-community-jdk-"
+         graalvm-version
+         "_linux-"
+         myarch
+         "_bin.tar.gz")))
+
+(defn build-native-image
+  [arch resource-class]
+  (let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")
+        graalvm-url (make-graalvm-url arch)]
+    (ordered-map
+     :machine
+     {:image "ubuntu-2204:2023.10.1"
+      :resource_class resource-class}
+     :working_directory "/home/circleci/replikativ"
+     :environment {:GRAALVM_VERSION graalvm-version
+                   :DTHK_PLATFORM "linux"
+                   :DTHK_ARCH arch
+                   :PATH "/bin:/home/circleci/graalvm/bin:/home/circleci/clojure/bin:/home/circleci/bin"
+                   :JAVA_HOME "/home/circleci/graalvm/bin/java"}
+     :steps
+     [:checkout
+      {:restore_cache {:keys [cache-key]}}
+      (run "Install GraalVM"
+           (format "cd /home/circleci
+/bin/wget -O graalvm.tar.gz %s
+/bin/mkdir graalvm || true
+/bin/tar -xzf graalvm.tar.gz --directory graalvm --strip-components 1
+sudo update-alternatives --install /usr/bin/java java /home/circleci/graalvm/bin/java 0
+sudo update-alternatives --install /usr/bin/javac javac /home/circleci/graalvm/bin/javac 0
+sudo update-alternatives --set java /home/circleci/graalvm/bin/java
+sudo update-alternatives --set javac /home/circleci/graalvm/bin/javac"
+                   graalvm-url))
+      (run "Install Clojure"
+           "cd /home/circleci
+/bin/curl -sLO https://download.clojure.org/install/linux-install-1.11.1.1165.sh
+/bin/chmod +x linux-install-1.11.1.1165.sh
+./linux-install-1.11.1.1165.sh --prefix /home/circleci/clojure")
+      (run "Install Babashka"
+           "cd /home/circleci
+/bin/curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
+/bin/chmod +x install
+./install --dir /home/circleci/bin")
+      (run "Build native image"
+           "cd /home/circleci/replikativ
+bb ni-cli")
+      (run "Test native image"
+           "cd /home/circleci/replikativ
+bb test native-image")
+      {:persist_to_workspace
+       {:root "/home/circleci/"
+        :paths ["replikativ/dthk"]}}
+      {:save_cache
+       {:paths ["~/.m2" "~/graalvm"]
+        :key cache-key}}])))
+
+(defn release-native-image
+  [arch]
+  (let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")]
+    (ordered-map
+     :executor "tools/clojurecli"
+     :working_directory "/home/circleci/replikativ"
+     :environment {:DTHK_PLATFORM "linux"
+                   :DTHK_ARCH arch}
+     :steps
+     [:checkout
+      {:restore_cache {:keys [cache-key]}}
+      {:attach_workspace {:at "/home/circleci"}}
+      (run "Release native image"
+           "cd /home/circleci/replikativ
+bb release native-image")
+      {:persist_to_workspace
+       {:root "/home/circleci/"
+        :paths ["replikativ/dthk"]}}
+      {:save_cache
+       {:paths ["~/.m2" "~/graalvm"]
+        :key cache-key}}])))
+
+(defn make-config []
+  (ordered-map
+   :version 2.1
+   :orbs
+   {:tools "replikativ/clj-tools@0"}
+   :commands
+   {:setup-docker-buildx
+    {:steps
+     [{:run
+       {:name "Create multi-platform capable buildx builder"
+        :command
+        "docker run --privileged --rm tonistiigi/binfmt --install all\ndocker buildx create --name ci-builder --use"}}]}}
+   :jobs (ordered-map
+          :build-linux-amd64 (build-native-image "amd64" "large")
+          :build-linux-aarch64 (build-native-image "aarch64" "arm.large")
+          :release-linux-amd64 (release-native-image "amd64")
+          :release-linux-aarch64 (release-native-image "aarch64"))
+   :workflows (ordered-map
+               :version 2
+               :native-images
+               {:jobs ["build-linux-amd64"
+                       "build-linux-aarch64"
+                       {"release-linux-amd64"
+                        {:context ["dockerhub-deploy"
+                                   "github-token"]
+                         :filters {:branches {:only "main"}}
+                         :requires ["build-linux-amd64"]}}
+                       {"release-linux-aarch64"
+                        {:context ["dockerhub-deploy"
+                                   "github-token"]
+                         :filters {:branches {:only "main"}}
+                         :requires ["build-linux-aarch64"]}}]})))
+
+(def skip-config
+  {:skip-if-only [#"^doc\/.*"
+                  #"^bb\/.*"
+                  #"^dev\/.*"
+                  #"^examples\/.*"
+                  #"^test\/.*"
+                  #".*.md$"]})
+
+(defn get-changes
+  []
+  (-> (tasks/shell {:out :string} "git diff --name-only HEAD~1")
+      (:out)
+      (str/split-lines)))
+
+(defn irrelevant-change?
+  [change regexes]
+  (some? (some #(re-matches % change) regexes)))
+
+(defn anything-relevant?
+  [change-set regexes]
+  (some? (some #(not (irrelevant-change? % regexes)) change-set)))
+
+(defn main
+  []
+  (let [{:keys [skip-if-only]} skip-config
+        changed-files          (get-changes)]
+    (when (anything-relevant? changed-files skip-if-only)
+      (-> (make-config)
+          (yaml/generate-string :dumper-options {:flow-style :block})
+          println))))
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (main))
+
+(comment
+  (def changed-files (get-changes))
+  (anything-relevant? changed-files (:skip-if-only skip-config))
+  (-> (make-config) 
+      (yaml/generate-string :dumper-options {:flow-style :block})
+      println))
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -0,0 +1,56 @@
+macos_instance:
+  image: ghcr.io/cirruslabs/macos-monterey-base:latest
+
+build-task:
+  env:
+    DTHK_PLATFORM: macos
+    DTHK_ARCH: aarch64
+    INSTALL_DIR: ${HOME}
+    GRAALVM_VERSION: 22.0.2
+    GRAALVM_SLUG: 9.1
+    GRAALVM_HOME: ${HOME}/graalvm-community-openjdk-${GRAALVM_VERSION}+${GRAALVM_SLUG}/Contents/Home/
+    GITHUB_TOKEN: ENCRYPTED[0ea782cbbd99c2486b0f55b8eefcf91606a0805faaa3f35e55ecb75268b0046b874c856753f528ba689a9cdaa11e47c2]
+  script: |
+    set -euo pipefail
+
+    # install babashka
+    curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
+    chmod +x install
+    ./install --dir ${HOME}/.local/bin
+    export PATH=${HOME}/.local/bin:$PATH
+
+    # install clojure
+    curl -L -O https://github.com/clojure/brew-install/releases/latest/download/posix-install.sh
+    chmod +x posix-install.sh
+    sudo ./posix-install.sh
+
+    # install graalvm
+    pushd "$INSTALL_DIR" >/dev/null
+    curl -L -O https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-${GRAALVM_VERSION}/graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz
+    sudo xattr -r -d com.apple.quarantine graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz
+    tar -xzf graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz
+    popd >/dev/null
+
+    # prepare
+    export PATH=${GRAALVM_HOME}/bin:${PATH}
+    export JAVA_HOME=${GRAALVM_HOME}
+    sudo /usr/sbin/softwareupdate --install-rosetta --agree-to-license
+    java -version
+    ${GRAALVM_HOME}bin/native-image --version
+    echo $PATH
+    native-image --version
+
+    # compile
+    bb ni-cli
+
+    # test
+    file dthk
+    bb test native-image
+    bb test bb-pod
+
+    # upload artifact
+    if [[ $CIRRUS_RELEASE == "" ]]; then
+      bb release native-cli
+    fi
+  binaries_artifacts:
+    path: "dist/*"