From fbd92868883e0935a8416404fa3f0c395591e476 Mon Sep 17 00:00:00 2001 From: Pulkit Jain Date: Mon, 19 Feb 2024 18:12:23 +0530 Subject: [PATCH] Run Multiple kind Jobs on same testbed Signed-off-by: Pulkit Jain --- Makefile | 12 ++++++++++++ build/images/base/build.sh | 22 ++++++++++++++++++---- build/images/ovs/build.sh | 24 +++++++++++++++++++----- ci/jenkins/jobs/macros.yaml | 4 ++-- ci/jenkins/test.sh | 26 ++++++++++++++++++++------ hack/build-antrea-linux-all.sh | 17 ++++++++++++++--- hack/generate-manifest.sh | 3 +++ 7 files changed, 88 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 062c54bde49..b201208c694 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,10 @@ OVS_VERSION := $(shell head -n 1 build/images/deps/ovs-version) GO_VERSION := $(shell head -n 1 build/images/deps/go-version) CNI_BINARIES_VERSION := $(shell head -n 1 build/images/deps/cni-binaries-version) BUILD_TAG := $(shell build/images/build-tag.sh) +NANOSERVER_VERSION := $(shell head -n 1 build/images/deps/nanoserver-version) +WIN_BUILD_TAG := $(shell echo $(GO_VERSION) $(CNI_BINARIES_VERSION) $(NANOSERVER_VERSION)|md5sum|head -c 10) +WIN_OVS_VERSION := $(shell head -n 1 build/images/deps/ovs-version-windows) +WIN_BUILD_OVS_TAG := $(NANOSERVER_VERSION)-$(WIN_OVS_VERSION) GIT_HOOKS := $(shell find hack/git_client_side_hooks -type f -print) DOCKER_NETWORK ?= default TRIVY_TARGET_IMAGE ?= @@ -28,6 +32,14 @@ GOLANGCI_LINT_VERSION := v1.54.0 GOLANGCI_LINT_BINDIR := $(CURDIR)/.golangci-bin GOLANGCI_LINT_BIN := $(GOLANGCI_LINT_BINDIR)/$(GOLANGCI_LINT_VERSION)/golangci-lint +BUILD_TAG := +ifndef $(CUSTOM_BUILD_TAG) + BUILD_TAG = $(shell build/images/build-tag.sh) +else + BUILD_TAG = $(CUSTOM_BUILD_TAG) + DOCKER_IMG_VERSION = $(CUSTOM_BUILD_TAG) +endif + DOCKER_BUILD_ARGS := ifeq ($(NO_PULL),) DOCKER_BUILD_ARGS += --pull diff --git a/build/images/base/build.sh b/build/images/base/build.sh index 80ad459cf38..10ed579e0f1 100755 --- a/build/images/base/build.sh +++ b/build/images/base/build.sh @@ -29,13 +29,19 @@ Build the antrea base image. --push Push the built image to the registry --platform Target platform for the image if server is multi-platform capable --distro Target Linux distribution - --no-cache Do not use the local build cache nor the cached image from the registry" + --no-cache Do not use the local build cache nor the cached image from the registry + --build-tag Custom build tag for images." + +function print_usage { + echoerr "$_usage" +} PULL=false PUSH=false NO_CACHE=false PLATFORM="" DISTRO="ubuntu" +BUILD_TAG="" while [[ $# -gt 0 ]] do @@ -62,6 +68,10 @@ case $key in NO_CACHE=true shift ;; + --build-tag) + BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -102,7 +112,11 @@ pushd $THIS_DIR > /dev/null CNI_BINARIES_VERSION=$(head -n 1 ../deps/cni-binaries-version) SURICATA_VERSION=$(head -n 1 ../deps/suricata-version) -BUILD_TAG=$(../build-tag.sh) +BUILD_CACHE_TAG=$(../build-tag.sh) + +if [[ $BUILD_TAG == "" ]]; then + BUILD_TAG=$BUILD_CACHE_TAG +fi if $PULL; then # The ubuntu image is also used for the UBI build (for the cni-binaries intermediate image). @@ -141,12 +155,12 @@ function docker_build_and_push() { local build_args="--build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg SURICATA_VERSION=$SURICATA_VERSION --build-arg BUILD_TAG=$BUILD_TAG" local cache_args="" if $PUSH; then - cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_TAG,mode=max" + cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max" fi if $NO_CACHE; then cache_args="$cache_args --no-cache" else - cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max" + cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max" fi docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile . diff --git a/build/images/ovs/build.sh b/build/images/ovs/build.sh index 7ac80c43e67..dc3335cae51 100755 --- a/build/images/ovs/build.sh +++ b/build/images/ovs/build.sh @@ -28,14 +28,20 @@ Build the antrea openvswitch image. --pull Always attempt to pull a newer version of the base images --push Push the built image to the registry --platform Target platform for the image if server is multi-platform capable - --distro Target distribution. If distro is 'windows', platform should be empty. The script uses 'windows/amd64' automatically - --no-cache Do not use the local build cache nor the cached image from the registry" + --distro Target Linux distribution + --no-cache Do not use the local build cache nor the cached image from the registry + --build-tag Custom build tag for images." + +function print_usage { + echoerr "$_usage" +} PULL=false PUSH=false NO_CACHE=false PLATFORM="" DISTRO="ubuntu" +BUILD_TAG="" while [[ $# -gt 0 ]] do @@ -62,6 +68,10 @@ case $key in NO_CACHE=true shift ;; + --build-tag) + BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -106,7 +116,11 @@ pushd $THIS_DIR > /dev/null OVS_VERSION=$(head -n 1 ${OVS_VERSION_FILE}) -BUILD_TAG=$(../build-tag.sh) +BUILD_CACHE_TAG=$(../build-tag.sh) + +if [[ $BUILD_TAG == "" ]]; then + BUILD_TAG=$BUILD_CACHE_TAG +fi if $PULL; then if [ "$DISTRO" == "ubuntu" ]; then @@ -130,12 +144,12 @@ function docker_build_and_push() { local build_args="--build-arg OVS_VERSION=$OVS_VERSION" local cache_args="" if $PUSH; then - cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_TAG,mode=max" + cache_args="$cache_args --cache-to type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max" fi if $NO_CACHE; then cache_args="$cache_args --no-cache" else - cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_TAG,mode=max" + cache_args="$cache_args --cache-from type=registry,ref=$image-cache:$BUILD_CACHE_TAG,mode=max" fi docker buildx build $PLATFORM_ARG -o type=docker -t $image:$BUILD_TAG $cache_args $build_args -f $dockerfile . diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml index 56358df2cb4..1afdb254e14 100644 --- a/ci/jenkins/jobs/macros.yaml +++ b/ci/jenkins/jobs/macros.yaml @@ -224,7 +224,7 @@ ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}' kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config set +e - ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" + ./ci/jenkins/test.sh --testcase '{conformance_type}' --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "${conformance_type}-${{BUILD_NUMBER}}" return_code=$? set -ex ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}" @@ -248,7 +248,7 @@ ./ci/kind/kind-setup.sh --antrea-cni create "${{FULL_CLUSTER_NAME}}" --ip-family '{ip_version}' kind export kubeconfig -n "${{FULL_CLUSTER_NAME}}" --kubeconfig ${{PWD}}/.kube/config set +e - ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" + ./ci/jenkins/test.sh --testcase e2e --registry ${{DOCKER_REGISTRY}} --kubeconfig ${{PWD}}/.kube/config --testbed-type "kind" --kind-cluster-name "${{FULL_CLUSTER_NAME}}" --build-tag "e2e-${{BUILD_NUMBER}}" return_code=$? set -ex ./ci/kind/kind-setup.sh destroy "${{FULL_CLUSTER_NAME}}" diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index 8c8c40d3acf..44605384bb9 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -55,6 +55,8 @@ CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=4h" CLEAN_STALE_IMAGES_CONTAINERD="crictl rmi --prune" PRINT_DOCKER_STATUS="docker system df -v" PRINT_CONTAINERD_STATUS="crictl ps --state Exited" +BUILD_TAG="latest" +MANIFEST_ARGS="" _usage="Usage: $0 [--kubeconfig ] [--workdir ] [--testcase ] @@ -69,7 +71,8 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o --ip-mode IP mode for flexible-ipam e2e test. Default is $DEFAULT_IP_MODE. It can also be ipv6 or ds. --win-image-node Name of the windows image node in cluster. Images are built by docker on this node. --kind-cluster-name Name of the kind Cluster. - --proxyall Enable proxyAll to test AntreaProxy." + --proxyall Enable proxyAll to test AntreaProxy. + --build-tag Custom build tag for images." function print_usage { echoerr "$_usage" @@ -120,6 +123,10 @@ case $key in WIN_IMAGE_NODE="$2" shift 2 ;; + --build-tag) + BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -521,7 +528,13 @@ function deliver_antrea { fi chmod -R g-w build/images/ovs chmod -R g-w build/images/base - DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + if [[ "$BUILD_TAG" != "latest" ]]; then + DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --build-tag ${BUILD_TAG} --pull + IMG_TAG="${BUILD_TAG}" ./hack/generate-manifest.sh $MANIFEST_ARGS + else + DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull + ./hack/generate-manifest.sh $MANIFEST_ARGS + fi make flow-aggregator-image # Enable verbose log for troubleshooting. @@ -575,8 +588,8 @@ function deliver_antrea { ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/antrea-ubuntu.tar; ctr -n=k8s.io images import ${DEFAULT_WORKDIR}/flow-aggregator.tar" || true done elif [[ $TESTBED_TYPE == "kind" ]]; then - kind load docker-image antrea/antrea-agent-ubuntu:latest --name ${KIND_CLUSTER} - kind load docker-image antrea/antrea-controller-ubuntu:latest --name ${KIND_CLUSTER} + kind load docker-image antrea/antrea-agent-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER} + kind load docker-image antrea/antrea-controller-ubuntu:$BUILD_TAG --name ${KIND_CLUSTER} kind load docker-image antrea/flow-aggregator:latest --name ${KIND_CLUSTER} kubectl config use-context kind-${KIND_CLUSTER} docker cp ./build/yamls/antrea.yml ${KIND_CLUSTER}-control-plane:/root/antrea.yml @@ -963,12 +976,13 @@ EOF export KUBECONFIG=${KUBECONFIG_PATH} if [[ $TESTBED_TYPE == "flexible-ipam" ]]; then - ./hack/generate-manifest.sh --flexible-ipam --multicast --verbose-log > build/yamls/antrea.yml + MANIFEST_ARGS="$MANIFEST_ARGS --flexible-ipam --multicast --verbose-log" fi if [[ $TESTCASE =~ "multicast" ]]; then - ./hack/generate-manifest.sh --encap-mode noEncap --multicast --multicast-interfaces "ens224" --verbose-log > build/yamls/antrea.yml + MANIFEST_ARGS="$MANIFEST_ARGS --encap-mode noEncap --multicast --multicast-interfaces ens224 --verbose-log" fi +echo $MANIFEST_ARGS source $WORKSPACE/ci/jenkins/utils.sh check_and_upgrade_golang diff --git a/hack/build-antrea-linux-all.sh b/hack/build-antrea-linux-all.sh index a87e922513b..cca6f117e78 100755 --- a/hack/build-antrea-linux-all.sh +++ b/hack/build-antrea-linux-all.sh @@ -29,7 +29,12 @@ all Dockerfiles. --coverage Build the image with support for code coverage. --platform Target platform for the images if server is multi-platform capable. --distro Target Linux distribution. - --no-cache Do not use the local build cache nor the cached image from the registry." + --no-cache Do not use the local build cache nor the cached image from the registry. + --build-tag Custom build tag for images." + +function print_usage { + echoerr "$_usage" +} PULL=false PUSH=false @@ -37,6 +42,7 @@ NO_CACHE=false COVERAGE=false PLATFORM="" DISTRO="ubuntu" +BUILD_TAG="" while [[ $# -gt 0 ]] do @@ -67,6 +73,10 @@ case $key in NO_CACHE=true shift ;; + --build-tag) + export BUILD_TAG="$2" + shift 2 + ;; -h|--help) print_usage exit 0 @@ -116,8 +126,9 @@ fi CNI_BINARIES_VERSION=$(head -n 1 build/images/deps/cni-binaries-version) GO_VERSION=$(head -n 1 build/images/deps/go-version) -BUILD_TAG=$(build/images/build-tag.sh) -echo "BUILD_TAG: $BUILD_TAG" +if [ "$BUILD_TAG" != "" ]; then + ARGS="$ARGS --build-tag $BUILD_TAG" +fi # We pull all images ahead of time, instead of calling the independent build.sh # scripts with "--pull". We do not want to overwrite the antrea/openvswitch diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 4fdeb2eede9..534ca72e8bc 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -367,6 +367,9 @@ if [ "$MODE" == "dev" ]; then else HELM_VALUES+=("controllerImage.repository=$CONTROLLER_IMG_NAME") fi + if [ "$IMG_TAG" != "" ]; then + HELM_VALUES+=("agentImage.tag=$IMG_TAG,controllerImage.tag=$IMG_TAG") + fi if $VERBOSE_LOG; then HELM_VALUES+=("logVerbosity=4")