From 260bb4a1c5f23c94241147cb0bc0a8bf87e1f684 Mon Sep 17 00:00:00 2001
From: Morlay <morlay.null@gmail.com>
Date: Wed, 21 Oct 2020 17:37:17 +0800
Subject: [PATCH] build: setup buildx for all-in-ones

Signed-off-by: Morlay <morlay.null@gmail.com>
---
 .travis.yml                                 |   2 +-
 CONTRIBUTING.md                             |  38 +++++++
 cmd/all-in-one/Dockerfile                   |  59 +++++-----
 cmd/opentelemetry/cmd/all-in-one/Dockerfile |   4 +-
 scripts/travis/build-all-in-one-image.sh    | 120 ++++++++++++++------
 scripts/travis/setup-docker-buildx.sh       |  56 +++++++++
 6 files changed, 211 insertions(+), 68 deletions(-)
 create mode 100755 scripts/travis/setup-docker-buildx.sh

diff --git a/.travis.yml b/.travis.yml
index ffddbc460a5..7cb23e5fb0b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -67,7 +67,7 @@ script:
   - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
   - if [ "$TESTS" == true ]; then make test-ci ; else echo 'skipping tests'; fi
   - if [ "$PROTO_GEN_TEST" == true ]; then make proto && git diff --name-status --exit-code ; else echo 'skipping protoc validation'; fi
-  - if [ "$ALL_IN_ONE" == true ]; then make create-baseimg-debugimg && bash ./scripts/travis/build-all-in-one-image.sh ; else echo 'skipping all_in_one'; fi
+  - if [ "$ALL_IN_ONE" == true ]; then bash ./scripts/travis/setup-docker-buildx.sh && bash ./scripts/travis/build-all-in-one-image.sh ; else echo 'skipping all_in_one'; fi
   - if [ "$CROSSDOCK" == true ]; then travis_retry bash ./scripts/travis/build-crossdock.sh ; else echo 'skipping crossdock'; fi
   - if [ "$CROSSDOCK_OTEL" == true ]; then travis_retry make build-crossdock crossdock-otel ; else echo 'skipping OpenTelemetry crossdock'; fi
   - if [ "$DOCKER" == true ]; then bash ./scripts/travis/build-docker-images.sh ; else echo 'skipping build-docker-images'; fi
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cef2d4dafd7..0e75fe63200 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -173,3 +173,41 @@ Before merging a PR make sure:
 
 Merge the PR by using "Squash and merge" option on Github. Avoid creating merge commits.
 After the merge make sure referenced issues were closed.
+
+### Developing guidelines
+
+#### Building OCI Images for multiple arch (linux/arm64, linux/amd64)
+
+[OCI Images](https://github.com/opencontainers/image-spec/blob/master/spec.md) could be built and published by [buildx](https://github.com/docker/buildx), 
+it could be executed for local publish `all-in-one` images via:
+
+```shell
+TRAVIS_SECURE_ENV_VARS=true NAMESPACE=$(whoami) BRANCH=master ./scripts/travis/build-all-in-one-image.sh
+```
+
+more arch support only need to change `--platform=linux/amd64,linux/arm64,linux/s390x`
+
+if we want to execute this in local env, need to setup buildx:
+
+1. install docker cli plugin
+
+```shell
+$ export DOCKER_BUILDKIT=1
+$ docker build --platform=local -o . git://github.com/docker/buildx
+$ mkdir -p ~/.docker/cli-plugins
+$ mv buildx ~/.docker/cli-plugins/docker-buildx
+```
+(via https://github.com/docker/buildx#with-buildx-or-docker-1903, if docker issue, could check https://docs.docker.com/engine/install/linux-postinstall/#troubleshooting)
+
+2. install qemu for multi arch
+
+```shell
+$ docker run --privileged --rm tonistiigi/binfmt --install all
+```
+(via https://github.com/docker/buildx#building-multi-platform-images)
+
+3. create a builder
+
+```shell
+$ docker buildx create --use --name builder
+```
\ No newline at end of file
diff --git a/cmd/all-in-one/Dockerfile b/cmd/all-in-one/Dockerfile
index 2c7589f95d7..ca831995df4 100644
--- a/cmd/all-in-one/Dockerfile
+++ b/cmd/all-in-one/Dockerfile
@@ -1,8 +1,23 @@
-ARG base_image
-ARG debug_image
+ARG TARGET=release
 
-FROM $base_image AS release
-ARG TARGETARCH=amd64
+FROM golang:1.15-alpine as build
+ENV GOPATH /go
+
+RUN apk add --update --no-cache ca-certificates make git && \
+    go get github.com/go-delve/delve/cmd/dlv && \
+    cd /go/src/github.com/go-delve/delve && \
+    make install
+
+FROM golang:1.15-alpine AS base-image-debug
+
+COPY --from=build /go/bin/dlv /go/bin/dlv
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+FROM alpine:3.12 AS base-image-release
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+FROM base-image-${TARGET} as common
 
 # Agent zipkin.thrift compact
 EXPOSE 5775/udp
@@ -25,43 +40,27 @@ EXPOSE 14250
 # Web HTTP
 EXPOSE 16686
 
-COPY all-in-one-linux-$TARGETARCH /go/bin/all-in-one-linux
 COPY sampling_strategies.json /etc/jaeger/
 
 VOLUME ["/tmp"]
-ENTRYPOINT ["/go/bin/all-in-one-linux"]
-CMD ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"]
 
-FROM $debug_image AS debug
-ARG TARGETARCH=amd64
+FROM common AS release
 
-# Agent zipkin.thrift compact
-EXPOSE 5775/udp
-
-# Agent jaeger.thrift compact
-EXPOSE 6831/udp
-
-# Agent jaeger.thrift binary
-EXPOSE 6832/udp
+ARG TARGETARCH
+COPY all-in-one-linux-${TARGETARCH} /go/bin/all-in-one-linux
 
-# Agent config HTTP
-EXPOSE 5778
+ENTRYPOINT ["/go/bin/all-in-one-linux"]
+CMD ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"]
 
-# Collector HTTP
-EXPOSE 14268
+FROM common AS debug
 
-# Collector gRPC
-EXPOSE 14250
-
-# Web HTTP
-EXPOSE 16686
+ARG TARGETARCH
+COPY all-in-one-debug-linux-${TARGETARCH} /go/bin/all-in-one-linux
 
 # Delve
 EXPOSE 12345
 
-COPY all-in-one-debug-linux-$TARGETARCH /go/bin/all-in-one-linux
-COPY sampling_strategies.json /etc/jaeger/
-
-VOLUME ["/tmp"]
 ENTRYPOINT ["/go/bin/dlv", "exec", "/go/bin/all-in-one-linux", "--headless", "--listen=:12345", "--api-version=2", "--accept-multiclient", "--log", "--"]
 CMD ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"]
+
+FROM ${TARGET}
\ No newline at end of file
diff --git a/cmd/opentelemetry/cmd/all-in-one/Dockerfile b/cmd/opentelemetry/cmd/all-in-one/Dockerfile
index 67d7a8913c8..f312e8bd850 100644
--- a/cmd/opentelemetry/cmd/all-in-one/Dockerfile
+++ b/cmd/opentelemetry/cmd/all-in-one/Dockerfile
@@ -2,9 +2,9 @@ FROM alpine:latest as certs
 RUN apk add --update --no-cache ca-certificates
 
 FROM scratch
-ARG TARGETARCH=amd64
 
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
-COPY opentelemetry-all-in-one-linux-$TARGETARCH /go/bin/opentelemetry-all-in-one-linux
+ARG TARGETARCH
+COPY opentelemetry-all-in-one-linux-${TARGETARCH} /go/bin/opentelemetry-all-in-one-linux
 ENTRYPOINT ["/go/bin/opentelemetry-all-in-one-linux"]
diff --git a/scripts/travis/build-all-in-one-image.sh b/scripts/travis/build-all-in-one-image.sh
index 8b390da13e7..60ecba3f649 100755
--- a/scripts/travis/build-all-in-one-image.sh
+++ b/scripts/travis/build-all-in-one-image.sh
@@ -7,6 +7,9 @@ BRANCH=${BRANCH:?'missing BRANCH env var'}
 # be overrided by passing architecture value to the script:
 # `GOARCH=<target arch> ./scripts/travis/build-all-in-one-image.sh`.
 GOARCH=${GOARCH:-$(go env GOARCH)}
+# local run with `TRAVIS_SECURE_ENV_VARS=true NAMESPACE=$(whoami) BRANCH=master ./scripts/travis/build-all-in-one-image.sh`
+NAMESPACE=${NAMESPACE:-jaegertracing}
+ARCHS="amd64 arm64 s390x"
 
 source ~/.nvm/nvm.sh
 nvm use 10
@@ -26,42 +29,89 @@ run_integration_test() {
   docker kill $CID
 }
 
-upload_to_docker() {
-  # Only push the docker container to Docker Hub for master branch
-  if [[ ("$BRANCH" == "master" || $BRANCH =~ ^v[0-9]+\.[0-9]+\.[0-9]+$) && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-    echo 'upload to Docker Hub'
+docker_buildx() {
+  CMD_ROOT=${1}
+  FLAGS=${@:2}
+
+  if [[ "$FLAGS" == *"-push"*  ]]; then
+    if [[ ("$BRANCH" == "master" || $BRANCH =~ ^v[0-9]+\.[0-9]+\.[0-9]+$) && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+      echo 'upload to Docker Hub'
+    else
+      echo 'skip docker upload for PR'
+      exit 0
+    fi
+  fi
+
+  docker buildx build -f ${CMD_ROOT}/Dockerfile ${FLAGS} ${CMD_ROOT}
+}
+
+image_tags_for() {
+  REPO=${1}
+
+  major=""
+  minor=""
+  patch=""
+
+  if [[ "$BRANCH" == "master" ]]; then
+    TAG="latest"
+  elif [[ $BRANCH =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
+    major="${BASH_REMATCH[1]}"
+    minor="${BASH_REMATCH[2]}"
+    patch="${BASH_REMATCH[3]}"
+    TAG=${major}.${minor}.${patch}
   else
-    echo 'skip docker upload for PR'
-    exit 0
+    TAG="${BRANCH///}"
+  fi
+
+  IMAGE_TAGS="--tag ${REPO}:${TAG}"
+
+  # add major, major.minor and major.minor.patch tags
+  if [[ -n $major ]]; then
+    IMAGE_TAGS="${IMAGE_TAGS} -t ${REPO}:${major}"
+    if [[ -n $minor ]]; then
+       IMAGE_TAGS="${IMAGE_TAGS} -t ${REPO}:${major}.${minor}"
+      if [[ -n $patch ]]; then
+          IMAGE_TAGS="${IMAGE_TAGS} -t ${REPO}:${major}.${minor}.${patch}"
+      fi
+    fi
   fi
-  export REPO=$1
-  bash ./scripts/travis/upload-to-docker.sh
+
+  echo "${IMAGE_TAGS}"
+}
+
+target_platforms() {
+  PLATFORMS=""
+  for arch in ${ARCHS}; do
+    if [ -n "${PLATFORMS}" ]; then
+      PLATFORMS="${PLATFORMS},linux/${arch}"
+    else
+      PLATFORMS="linux/${arch}"
+    fi
+  done
+  echo ${PLATFORMS}
 }
 
-make build-all-in-one GOOS=linux GOARCH=$GOARCH
-repo=jaegertracing/all-in-one
-docker build -f cmd/all-in-one/Dockerfile \
-        --target release \
-        --tag $repo:latest cmd/all-in-one \
-        --build-arg base_image=localhost/baseimg:1.0.0-alpine-3.12 \
-        --build-arg debug_image=localhost/debugimg:1.0.0-golang-1.15-alpine \
-        --build-arg TARGETARCH=$GOARCH
-run_integration_test $repo
-upload_to_docker $repo
-
-make build-all-in-one-debug GOOS=linux GOARCH=$GOARCH
-repo=jaegertracing/all-in-one-debug
-docker build -f cmd/all-in-one/Dockerfile \
-        --target debug \
-        --tag $repo:latest cmd/all-in-one \
-        --build-arg base_image=localhost/baseimg:1.0.0-alpine-3.12 \
-        --build-arg debug_image=localhost/debugimg:1.0.0-golang-1.15-alpine \
-        --build-arg TARGETARCH=$GOARCH
-run_integration_test $repo
-upload_to_docker $repo
-
-make build-otel-all-in-one GOOS=linux GOARCH=$GOARCH
-repo=jaegertracing/opentelemetry-all-in-one
-docker build -f cmd/opentelemetry/cmd/all-in-one/Dockerfile -t $repo:latest cmd/opentelemetry/cmd/all-in-one --build-arg TARGETARCH=$GOARCH
-run_integration_test $repo
-upload_to_docker $repo
+
+for arch in ${ARCHS}; do
+  make build-all-in-one GOOS=linux GOARCH=${arch}
+done
+repo=${NAMESPACE}/all-in-one
+docker_buildx cmd/all-in-one --load --build-arg=TARGET=release --platform=linux/${GOARCH} -t $repo:latest
+run_integration_test ${repo}
+docker_buildx cmd/all-in-one --push --build-arg=TARGET=release --platform=$(target_platforms) $(image_tags_for ${repo})
+
+for arch in ${ARCHS}; do
+  make build-all-in-one-debug GOOS=linux GOARCH=${arch}
+done
+repo=${NAMESPACE}/all-in-one-debug
+docker_buildx cmd/all-in-one --load --build-arg=TARGET=debug --platform=linux/${GOARCH} -t $repo:latest
+run_integration_test ${repo}
+docker_buildx cmd/all-in-one --push --build-arg=TARGET=debug --platform=$(target_platforms) $(image_tags_for ${repo})
+
+for arch in ${ARCHS}; do
+  make build-otel-all-in-one GOOS=linux GOARCH=${arch}
+done
+repo=${NAMESPACE}/opentelemetry-all-in-one
+docker_buildx cmd/opentelemetry/cmd/all-in-one --load --platform=linux/${GOARCH} -t $repo:latest
+run_integration_test ${repo}
+docker_buildx cmd/opentelemetry/cmd/all-in-one --push --platform=$(target_platforms) $(image_tags_for ${repo})
\ No newline at end of file
diff --git a/scripts/travis/setup-docker-buildx.sh b/scripts/travis/setup-docker-buildx.sh
new file mode 100755
index 00000000000..8398525ec91
--- /dev/null
+++ b/scripts/travis/setup-docker-buildx.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -eux
+
+# update docker
+# for setup buildx https://travis-ci.community/t/installing-docker-19-03/8077/2
+sudo apt update
+
+sudo systemctl stop docker
+sudo apt install -y docker.io
+
+sudo systemctl unmask docker.service
+sudo systemctl unmask docker.socket
+sudo systemctl start docker
+sudo systemctl status docker.socket
+
+docker version
+
+DOCKER_BUILDX_VERSION=v0.4.2
+
+LOCAL_OS=$(uname -s | tr '[A-Z]' '[a-z]')
+
+case $(uname -m) in
+x86_64)
+  LOCAL_ARCH=amd64
+  ;;
+aarch64)
+  LOCAL_ARCH=arm64
+  ;;
+*)
+  echo "unsupported architecture"
+  exit 1
+  ;;
+esac
+
+if [[ ! -f ~/.docker/cli-plugins/docker-buildx ]]; then
+  DOCKER_BUILDX_DOWNLOAD_URL=https://github.com/docker/buildx/releases/download/${DOCKER_BUILDX_VERSION}/buildx-${DOCKER_BUILDX_VERSION}.${LOCAL_OS}-${LOCAL_ARCH}
+  mkdir -p ~/.docker/cli-plugins
+  echo "downloading from ${DOCKER_BUILDX_DOWNLOAD_URL}"
+  curl -sL --output ~/.docker/cli-plugins/docker-buildx "${DOCKER_BUILDX_DOWNLOAD_URL}"
+  chmod a+x ~/.docker/cli-plugins/docker-buildx
+fi
+
+# enable buildx
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+# checkout buildx available
+docker buildx version
+
+# enabled qemu if needed
+if [[ ! $(docker buildx inspect default | grep Platforms) == *arm64* ]]; then
+  docker run --privileged --rm tonistiigi/binfmt --install all
+fi
+
+# setup builder
+docker buildx create --use