chore: add latest-official docker tag (#25322)

.github/workflows/docker-release.yml

@@ -19,4 +19,5 @@ jobs:
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
-          .github/workflows/docker_build_push.sh
+          GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
+          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME"

.github/workflows/docker.yml

@@ -41,7 +41,7 @@ jobs:
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
-          .github/workflows/docker_build_push.sh
+          ./scripts/docker_build_push.sh
 
       - name: Build ephemeral env image
         if: github.event_name == 'pull_request'

.github/workflows/docker_build_push.sh → scripts/docker_build_push.sh

@@ -17,6 +17,8 @@
 #
 set -eo pipefail
 
+GITHUB_RELEASE_TAG_NAME="$1"
+
 SHA=$(git rev-parse HEAD)
 REPO_NAME="apache/superset"
 
@@ -32,17 +34,75 @@ else
   LATEST_TAG="${REFSPEC}"
 fi
 
+
 if [[ "${REFSPEC}" == "master" ]]; then
-  LATEST_TAG="latest"
+  LATEST_TAG="master"
+fi
+
+# get the latest release tag
+if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then
+  output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true
+  SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2)
+  if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then
+    LATEST_TAG="latest"
+  fi
+fi
+
+if [[ "${TEST_ENV}" == "true" ]]; then
+  # don't run the build in test environment
+  echo "LATEST_TAG is ${LATEST_TAG}"
+  exit 0
 fi
 
+
 cat<<EOF
   Rolling with tags:
   - ${REPO_NAME}:${SHA}
   - ${REPO_NAME}:${REFSPEC}
   - ${REPO_NAME}:${LATEST_TAG}
 EOF
 
+if [ -z "${DOCKERHUB_TOKEN}" ]; then
+  # Skip if secrets aren't populated -- they're only visible for actions running in the repo (not on forks)
+  echo "Skipping Docker push"
+  # By default load it back
+  DOCKER_ARGS="--load"
+  ARCHITECTURE_FOR_BUILD="linux/amd64 linux/arm64"
+else
+  # Login and push
+  docker logout
+  docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
+  DOCKER_ARGS="--push"
+  ARCHITECTURE_FOR_BUILD="linux/amd64,linux/arm64"
+fi
+set -x
+
+# for the dev image, it's ok to tag master as latest-dev
+# for production, we only want to tag the latest official release as latest
+if [ "${LATEST_TAG}" = "master" ]; then
+  DEV_TAG="${REPO_NAME}:latest-dev"
+else
+  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+fi
+
+#
+# Build the dev image
+#
+docker buildx build --target dev \
+  $DOCKER_ARGS \
+  --cache-from=type=registry,ref=apache/superset:master-dev \
+  --cache-from=type=local,src=/tmp/superset \
+  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
+  -t "${REPO_NAME}:${SHA}-dev" \
+  -t "${REPO_NAME}:${REFSPEC}-dev" \
+  -t "${DEV_TAG}" \
+  --platform linux/amd64 \
+  --label "sha=${SHA}" \
+  --label "built_at=$(date)" \
+  --label "target=dev" \
+  --label "build_actor=${GITHUB_ACTOR}" \
+  .
+
 #
 # Build the "lean" image
 #
@@ -71,6 +131,26 @@ docker build --target lean \
   --label "build_actor=${GITHUB_ACTOR}" \
   .
 
+#
+# Build the "lean39" image
+#
+docker buildx build --target lean \
+  $DOCKER_ARGS \
+  --cache-from=type=local,src=/tmp/superset \
+  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
+  -t "${REPO_NAME}:${SHA}-py39" \
+  -t "${REPO_NAME}:${REFSPEC}-py39" \
+  -t "${REPO_NAME}:${LATEST_TAG}-py39" \
+  --platform linux/amd64 \
+  --build-arg PY_VER="3.9-slim-bullseye"\
+  --label "sha=${SHA}" \
+  --label "built_at=$(date)" \
+  --label "target=lean39" \
+  --label "build_actor=${GITHUB_ACTOR}" \
+  .
+
+
+for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
 #
 # Build the "websocket" image
 #

scripts/tag_latest_release.sh

@@ -17,15 +17,63 @@
 #
 
 run_git_tag () {
-  if [ "$DRY_RUN" = "false" ] && [ "$SKIP_TAG" = "false" ]
+  if [[ "$DRY_RUN" == "false" ]] && [[ "$SKIP_TAG" == "false" ]]
   then
     git tag -a -f latest "${GITHUB_TAG_NAME}" -m "latest tag"
     echo "${GITHUB_TAG_NAME} has been tagged 'latest'"
   fi
   exit 0
 }
 
-echo "::set-output name=SKIP_TAG::false"
+###
+# separating out git commands into functions so they can be mocked in unit tests
+###
+git_show_ref () {
+  if [[ "$TEST_ENV" == "true" ]]
+  then
+    if [[ "$GITHUB_TAG_NAME" == "does_not_exist" ]]
+        # mock return for testing only
+    then
+      echo ""
+    else
+      echo "2817aebd69dc7d199ec45d973a2079f35e5658b6 refs/tags/${GITHUB_TAG_NAME}"
+    fi
+  fi
+  result=$(git show-ref "${GITHUB_TAG_NAME}")
+  echo "${result}"
+}
+
+get_latest_tag_list () {
+  if [[ "$TEST_ENV" == "true" ]]
+  then
+    echo "(tag: 2.1.0, apache/2.1test)"
+  else
+    result=$(git show-ref --tags --dereference latest | awk '{print $2}' | xargs git show --pretty=tformat:%d -s | grep tag:)
+    echo "${result}"
+  fi
+}
+###
+
+split_string () {
+  local version="$1"
+  local delimiter="$2"
+  local components=()
+  local tmp=""
+  for (( i=0; i<${#version}; i++ )); do
+    local char="${version:$i:1}"
+    if [[ "$char" != "$delimiter" ]]; then
+      tmp="$tmp$char"
+    elif [[ -n "$tmp" ]]; then
+      components+=("$tmp")
+      tmp=""
+    fi
+  done
+  if [[ -n "$tmp" ]]; then
+    components+=("$tmp")
+  fi
+  echo "${components[@]}"
+}
+
 DRY_RUN=false
 
 # get params passed in with script when it was run
@@ -50,49 +98,56 @@ done
 
 if [ -z "${GITHUB_TAG_NAME}" ]; then
     echo "Missing tag parameter, usage: ./scripts/tag_latest_release.sh <GITHUB_TAG_NAME>"
+    echo "::set-output name=SKIP_TAG::true"
     exit 1
 fi
 
-if [ -z "$(git show-ref ${GITHUB_TAG_NAME})" ]; then
+if [ -z "$(git_show_ref)" ]; then
     echo "The tag ${GITHUB_TAG_NAME} does not exist. Please use a different tag."
-    exit 1
+    echo "::set-output name=SKIP_TAG::true"
+    exit 0
 fi
 
 # check that this tag only contains a proper semantic version
 if ! [[ ${GITHUB_TAG_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
 then
   echo "This tag ${GITHUB_TAG_NAME} is not a valid release version. Not tagging."
   echo "::set-output name=SKIP_TAG::true"
-  exit 0
+  exit 1
 fi
 
 ## split the current GITHUB_TAG_NAME into an array at the dot
-IFS=$'.'
-THIS_TAG_NAME=(${GITHUB_TAG_NAME})  || echo 'not found'
+THIS_TAG_NAME=$(split_string "${GITHUB_TAG_NAME}" ".")
 
 # look up the 'latest' tag on git
-LATEST_TAG_LIST=$(git show-ref latest && git show --pretty=tformat:%d -s latest | grep tag:) || echo 'not found'
+LATEST_TAG_LIST=$(get_latest_tag_list) || echo 'not found'
 
 # if 'latest' tag doesn't exist, then set this commit to latest
 if [[ -z "$LATEST_TAG_LIST" ]]
 then
-  # move on to next task
   echo "there are no latest tags yet, so I'm going to start by tagging this sha as the latest"
   run_git_tag
+  exit 0
 fi
 
-## get all tags that use the same sha as the latest tag. split at comma.
-IFS=$','
-LATEST_TAGS=($LATEST_TAG_LIST)
+# remove parenthesis and tag: from the list of tags
+LATEST_TAGS_STRINGS=$(echo "$LATEST_TAG_LIST" | sed 's/tag: \([^,]*\)/\1/g' | tr -d '()')
 
-## loop over those tags and only take action on the one that isn't tagged 'latest'
-## that one will have the version number tag
-for (( i=0; i<${#LATEST_TAGS[@]}; i++ ))
+LATEST_TAGS=$(split_string "$LATEST_TAGS_STRINGS" ",")
+TAGS=($(split_string "$LATEST_TAGS" " "))
+
+# Initialize a flag for comparison result
+compare_result=""
+
+# Iterate through the tags of the latest release
+for tag in $TAGS
 do
-  if [[ ${LATEST_TAGS[$i]} != *"latest"* ]]
-  then
+  if [[ $tag == "latest" ]]; then
+    continue
+  else
     ## extract just the version from this tag
-    LATEST_RELEASE_TAG=$(echo "${LATEST_TAGS[$i]}" | sed -E -e 's/tag:|\(|\)|[[:space:]]*//g')
+    LATEST_RELEASE_TAG="$tag"
+    echo "LATEST_RELEASE_TAG: ${LATEST_RELEASE_TAG}"
 
     # check that this only contains a proper semantic version
     if ! [[ ${LATEST_RELEASE_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
@@ -101,28 +156,35 @@ do
       continue
     fi
     echo "The current release with the latest tag is version ${LATEST_RELEASE_TAG}"
-
-    ## remove the sha from the latest tag and split into an array- split at the dot
-    IFS=$'.'
-    LATEST_RELEASE_TAG_SPLIT=(${LATEST_RELEASE_TAG})
-
-    for (( j=0; j<${#THIS_TAG_NAME[@]}; j++ ))
-    do
-      ## if this value is greater than the latest release, then tag it, if it's lower, then stop, if it's
-      ## the same then move on to the next index
-      if [[ ${THIS_TAG_NAME[$j]} -gt ${LATEST_RELEASE_TAG_SPLIT[$j]} ]]
-      then
-        echo "This release tag ${GITHUB_TAG_NAME} is the latest. Tagging it"
-        run_git_tag
-
-      elif [[ ${THIS_TAG_NAME[$j]} -lt ${LATEST_RELEASE_TAG_SPLIT[$j]} ]]
-      then
-        continue
-      fi
+    # Split the version strings into arrays
+    THIS_TAG_NAME_ARRAY=($(split_string "$THIS_TAG_NAME" "."))
+    LATEST_RELEASE_TAG_ARRAY=($(split_string "$LATEST_RELEASE_TAG" "."))
+
+    # Iterate through the components of the version strings
+    for (( j=0; j<${#THIS_TAG_NAME_ARRAY[@]}; j++ )); do
+        echo "Comparing ${THIS_TAG_NAME_ARRAY[$j]} to ${LATEST_RELEASE_TAG_ARRAY[$j]}"
+        if [[ $((THIS_TAG_NAME_ARRAY[$j])) > $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then
+            compare_result="greater"
+            break
+        elif [[ $((THIS_TAG_NAME_ARRAY[$j])) < $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then
+            compare_result="lesser"
+            break
+        fi
     done
   fi
 done
 
-echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging."
-# if you've gotten this far, then we don't want to run any tags in the next step
-echo "::set-output name=SKIP_TAG::true"
+# Determine the result based on the comparison
+if [[ -z "$compare_result" ]]; then
+    echo "Versions are equal"
+    echo "::set-output name=SKIP_TAG::true"
+elif [[ "$compare_result" == "greater" ]]; then
+    echo "This release tag ${GITHUB_TAG_NAME} is newer than the latest."
+    echo "::set-output name=SKIP_TAG::false"
+    # Add other actions you want to perform for a newer version
+elif [[ "$compare_result" == "lesser" ]]; then
+    echo "This release tag ${GITHUB_TAG_NAME} is older than the latest."
+    echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging."
+    # if you've gotten this far, then we don't want to run any tags in the next step
+    echo "::set-output name=SKIP_TAG::true"
+fi

tests/unit_tests/fixtures/bash_mock.py

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+
+
+class BashMock:
+    @staticmethod
+    def tag_latest_release(tag):
+        bash_command = f"./scripts/tag_latest_release.sh {tag} --dry-run"
+        result = subprocess.run(
+            bash_command,
+            shell=True,
+            capture_output=True,
+            text=True,
+            env={"TEST_ENV": "true"},
+        )
+        return result
+
+    @staticmethod
+    def docker_build_push(tag, branch):
+        bash_command = f"./scripts/docker_build_push.sh {tag}"
+        result = subprocess.run(
+            bash_command,
+            shell=True,
+            capture_output=True,
+            text=True,
+            env={"TEST_ENV": "true", "GITHUB_REF": f"refs/heads/{branch}"},
+        )
+        return result