Run Multiple kind Jobs on same testbed

Signed-off-by: Pulkit Jain <pulkit.jain@broadcom.com>
antrea-io · Jun 5, 2024 · fbd9286 · fbd9286
1 parent d895124
commit fbd9286
Show file tree

Hide file tree

Showing 7 changed files with 88 additions and 20 deletions.
diff --git a/Makefile b/Makefile
@@ -20,6 +20,10 @@ OVS_VERSION        := $(shell head -n 1 build/images/deps/ovs-version)
 GO_VERSION         := $(shell head -n 1 build/images/deps/go-version)
 CNI_BINARIES_VERSION := $(shell head -n 1 build/images/deps/cni-binaries-version)
 BUILD_TAG          := $(shell build/images/build-tag.sh)
+NANOSERVER_VERSION := $(shell head -n 1 build/images/deps/nanoserver-version)
+WIN_BUILD_TAG      := $(shell echo $(GO_VERSION) $(CNI_BINARIES_VERSION) $(NANOSERVER_VERSION)|md5sum|head -c 10)
+WIN_OVS_VERSION	   := $(shell head -n 1 build/images/deps/ovs-version-windows)
+WIN_BUILD_OVS_TAG  := $(NANOSERVER_VERSION)-$(WIN_OVS_VERSION)
 GIT_HOOKS          := $(shell find hack/git_client_side_hooks -type f -print)
 DOCKER_NETWORK     ?= default
 TRIVY_TARGET_IMAGE ?=
@@ -28,6 +32,14 @@ GOLANGCI_LINT_VERSION := v1.54.0
 GOLANGCI_LINT_BINDIR  := $(CURDIR)/.golangci-bin
 GOLANGCI_LINT_BIN     := $(GOLANGCI_LINT_BINDIR)/$(GOLANGCI_LINT_VERSION)/golangci-lint
 
+BUILD_TAG := 
+ifndef $(CUSTOM_BUILD_TAG)
+	BUILD_TAG = $(shell build/images/build-tag.sh)
+else
+	BUILD_TAG = $(CUSTOM_BUILD_TAG)
+	DOCKER_IMG_VERSION = $(CUSTOM_BUILD_TAG)
+endif
+
 DOCKER_BUILD_ARGS :=
 ifeq ($(NO_PULL),)
 	DOCKER_BUILD_ARGS += --pull

diff --git a/build/images/base/build.sh b/build/images/base/build.sh
@@ -29,13 +29,19 @@ Build the antrea base image.
         --push                  Push the built image to the registry
         --platform <PLATFORM>   Target platform for the image if server is multi-platform capable
         --distro <distro>       Target Linux distribution
-        --no-cache              Do not use the local build cache nor the cached image from the registry"
+        --no-cache              Do not use the local build cache nor the cached image from the registry
+        --build-tag             Custom build tag for images."
+
+function print_usage {
+    echoerr "$_usage"
+}
 
 PULL=false
 PUSH=false
 NO_CACHE=false
 PLATFORM=""
 DISTRO="ubuntu"
+BUILD_TAG=""
 
 while [[ $# -gt 0 ]]
 do
@@ -62,6 +68,10 @@ case $key in
     NO_CACHE=true
     shift
     ;;
+    --build-tag)
+    BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -102,7 +112,11 @@ pushd $THIS_DIR > /dev/null
 CNI_BINARIES_VERSION=$(head -n 1 ../deps/cni-binaries-version)
 SURICATA_VERSION=$(head -n 1 ../deps/suricata-version)
 
-BUILD_TAG=$(../build-tag.sh)
+BUILD_CACHE_TAG=$(../build-tag.sh)
+
+if [[ $BUILD_TAG == "" ]]; then
+    BUILD_TAG=$BUILD_CACHE_TAG
+fi
 
 if $PULL; then
     # The ubuntu image is also used for the UBI build (for the cni-binaries intermediate image).
@@ -141,12 +155,12 @@ function docker_build_and_push() {
     local build_args="--build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg SURICATA_VERSION=$SURICATA_VERSION --build-arg BUILD_TAG=$BUILD_TAG"
     local cache_args=""
     if $PUSH; then
-        cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
+        cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max"
     fi
     if $NO_CACHE; then
         cache_args="$cache_args --no-cache"
     else
-        cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
+        cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max"
     fi
     docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile .
 

diff --git a/build/images/ovs/build.sh b/build/images/ovs/build.sh
@@ -28,14 +28,20 @@ Build the antrea openvswitch image.
         --pull                  Always attempt to pull a newer version of the base images
         --push                  Push the built image to the registry
         --platform <PLATFORM>   Target platform for the image if server is multi-platform capable
-        --distro <distro>       Target distribution. If distro is 'windows', platform should be empty. The script uses 'windows/amd64' automatically
-        --no-cache              Do not use the local build cache nor the cached image from the registry"
+        --distro <distro>       Target Linux distribution
+        --no-cache              Do not use the local build cache nor the cached image from the registry
+        --build-tag             Custom build tag for images."
+
+function print_usage {
+    echoerr "$_usage"
+}
 
 PULL=false
 PUSH=false
 NO_CACHE=false
 PLATFORM=""
 DISTRO="ubuntu"
+BUILD_TAG=""
 
 while [[ $# -gt 0 ]]
 do
@@ -62,6 +68,10 @@ case $key in
     NO_CACHE=true
     shift
     ;;
+    --build-tag)
+    BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -106,7 +116,11 @@ pushd $THIS_DIR > /dev/null
 
 OVS_VERSION=$(head -n 1 ${OVS_VERSION_FILE})
 
-BUILD_TAG=$(../build-tag.sh)
+BUILD_CACHE_TAG=$(../build-tag.sh)
+
+if [[ $BUILD_TAG == "" ]]; then
+    BUILD_TAG=$BUILD_CACHE_TAG
+fi
 
 if $PULL; then
     if [ "$DISTRO" == "ubuntu" ]; then
@@ -130,12 +144,12 @@ function docker_build_and_push() {
     local build_args="--build-arg OVS_VERSION=$OVS_VERSION"
     local cache_args=""
     if $PUSH; then
-        cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
+        cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max"
     fi
     if $NO_CACHE; then
         cache_args="$cache_args --no-cache"
     else
-        cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max"
+        cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max"
     fi
     docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile .
 

diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml
@@ -224,7 +224,7 @@
           ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}'
           kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config
           set +e
-          ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}"
+          ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "${conformance_type}-${{BUILD_NUMBER}}"
           return_code=$?
           set -ex
           ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"
@@ -248,7 +248,7 @@
           ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}'
           kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config
           set +e
-          ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}"
+          ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "e2e-${{BUILD_NUMBER}}"
           return_code=$?
           set -ex
           ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}"

diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -55,6 +55,8 @@ CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=4h"
 CLEAN_STALE_IMAGES_CONTAINERD="crictl rmi --prune"
 PRINT_DOCKER_STATUS="docker system df -v"
 PRINT_CONTAINERD_STATUS="crictl ps --state Exited"
+BUILD_TAG="latest"
+MANIFEST_ARGS=""
 
 _usage="Usage: $0 [--kubeconfig <KubeconfigSavePath>] [--workdir <HomePath>]
                   [--testcase <windows-install-ovs|windows-conformance|windows-networkpolicy|windows-e2e|e2e|conformance|networkpolicy|multicast-e2e>]
@@ -69,7 +71,8 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o
         --ip-mode                IP mode for flexible-ipam e2e test. Default is $DEFAULT_IP_MODE. It can also be ipv6 or ds.
         --win-image-node         Name of the windows image node in cluster. Images are built by docker on this node.
         --kind-cluster-name      Name of the kind Cluster.
-        --proxyall               Enable proxyAll to test AntreaProxy."
+        --proxyall               Enable proxyAll to test AntreaProxy.
+        --build-tag              Custom build tag for images."
 
 function print_usage {
     echoerr "$_usage"
@@ -120,6 +123,10 @@ case $key in
     WIN_IMAGE_NODE="$2"
     shift 2
     ;;
+    --build-tag)
+    BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -521,7 +528,13 @@ function deliver_antrea {
     fi
     chmod -R g-w build/images/ovs
     chmod -R g-w build/images/base
-    DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
+    if [[ "$BUILD_TAG" != "latest" ]]; then
+        DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --build-tag ${BUILD_TAG} --pull
+        IMG_TAG="${BUILD_TAG}" ./hack/generate-manifest.sh $MANIFEST_ARGS
+    else
+        DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
+        ./hack/generate-manifest.sh $MANIFEST_ARGS
+    fi
     make flow-aggregator-image
 
     # Enable verbose log for troubleshooting.
@@ -575,8 +588,8 @@ function deliver_antrea {
             ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true
         done
     elif [[ $TESTBED_TYPE == "kind" ]]; then
-            kind load docker-image antrea/antrea-agent-ubuntu:latest --name ${KIND_CLUSTER}
-            kind load docker-image antrea/antrea-controller-ubuntu:latest --name ${KIND_CLUSTER}
+            kind load docker-image antrea/antrea-agent-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER}
+            kind load docker-image antrea/antrea-controller-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER}
             kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} 
             kubectl config use-context kind-${KIND_CLUSTER}
             docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml 
@@ -963,12 +976,13 @@ EOF
 
 export KUBECONFIG=${KUBECONFIG_PATH}
 if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then
-    ./hack/generate-manifest.sh --flexible-ipam --multicast --verbose-log > build/yamls/antrea.yml
+    MANIFEST_ARGS="$MANIFEST_ARGS --flexible-ipam --multicast --verbose-log"
 fi
 
 if [[ $TESTCASE =~ "multicast" ]]; then
-    ./hack/generate-manifest.sh --encap-mode noEncap --multicast --multicast-interfaces "ens224" --verbose-log > build/yamls/antrea.yml
+    MANIFEST_ARGS="$MANIFEST_ARGS --encap-mode noEncap --multicast --multicast-interfaces ens224 --verbose-log"
 fi
+echo $MANIFEST_ARGS
 
 source $WORKSPACE/ci/jenkins/utils.sh
 check_and_upgrade_golang

diff --git a/hack/build-antrea-linux-all.sh b/hack/build-antrea-linux-all.sh
@@ -29,14 +29,20 @@ all Dockerfiles.
         --coverage              Build the image with support for code coverage.
         --platform <PLATFORM>   Target platform for the images if server is multi-platform capable.
         --distro <distro>       Target Linux distribution.
-        --no-cache              Do not use the local build cache nor the cached image from the registry."
+        --no-cache              Do not use the local build cache nor the cached image from the registry.
+        --build-tag             Custom build tag for images."
+
+function print_usage {
+    echoerr "$_usage"
+}
 
 PULL=false
 PUSH=false
 NO_CACHE=false
 COVERAGE=false
 PLATFORM=""
 DISTRO="ubuntu"
+BUILD_TAG=""
 
 while [[ $# -gt 0 ]]
 do
@@ -67,6 +73,10 @@ case $key in
     NO_CACHE=true
     shift
     ;;
+    --build-tag)
+    export BUILD_TAG="$2"
+    shift 2
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -116,8 +126,9 @@ fi
 CNI_BINARIES_VERSION=$(head -n 1 build/images/deps/cni-binaries-version)
 GO_VERSION=$(head -n 1 build/images/deps/go-version)
 
-BUILD_TAG=$(build/images/build-tag.sh)
-echo "BUILD_TAG: $BUILD_TAG"
+if [ "$BUILD_TAG" != "" ]; then
+    ARGS="$ARGS --build-tag $BUILD_TAG"
+fi
 
 # We pull all images ahead of time, instead of calling the independent build.sh
 # scripts with "--pull". We do not want to overwrite the antrea/openvswitch

diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh
@@ -367,6 +367,9 @@ if [ "$MODE" == "dev" ]; then
     else
         HELM_VALUES+=("controllerImage.repository=$CONTROLLER_IMG_NAME")
     fi
+    if [ "$IMG_TAG" != "" ]; then
+        HELM_VALUES+=("agentImage.tag=$IMG_TAG,controllerImage.tag=$IMG_TAG")
+    fi
 
     if $VERBOSE_LOG; then
         HELM_VALUES+=("logVerbosity=4")