diff --git a/ci/jenkins/jobs/job-templates.yaml b/ci/jenkins/jobs/job-templates.yaml index 10b5829bf6e..4e9f52ca71a 100644 --- a/ci/jenkins/jobs/job-templates.yaml +++ b/ci/jenkins/jobs/job-templates.yaml @@ -284,6 +284,30 @@ publishers: '{publishers}' wrappers: '{wrappers}' +- job-template: + name: 'cloud-{name}-{test_name}-credential' + node: '{node}' + triggers: '{triggers}' + builders: '{builders}' + parameters: + - string: + default: '' + description: The cloud token ID of the cloud provider. + name: ID_TOKEN + trim: 'false' + description: '{description}' + block-downstream: false + block-upstream: false + project-type: freestyle + properties: + - build-discarder: + artifact-days-to-keep: -1 + artifact-num-to-keep: -1 + days-to-keep: 30 + num-to-keep: 30 + publishers: '{publishers}' + wrappers: '{wrappers}' + - job-template: name: 'cloud-{name}-{test_name}-cleanup' node: '{node}' diff --git a/ci/jenkins/jobs/projects-cloud.yaml b/ci/jenkins/jobs/projects-cloud.yaml index ca3d29e2293..2ed84bbeaba 100644 --- a/ci/jenkins/jobs/projects-cloud.yaml +++ b/ci/jenkins/jobs/projects-cloud.yaml @@ -718,10 +718,22 @@ builders: - shell: |- #!/bin/bash - # gcloud auth login requires verification code from url - ${{GCLOUD_SDK_PATH}}/bin/gcloud auth activate-service-account --key-file=${{GCLOUD_KEY_PATH}} + # Prevent exposing credentials in the console output by adding set +x + # This is to avoid other developers removing this set+x by accident + set +x + token_file="/var/lib/jenkins/gcp_cred" + cp $token_file ${{WORKSPACE}}/gcp_cred + sudo gcloud iam workforce-pools create-cred-config \ + locations/global/workforcePools/${{WORKFORCE_POOL}}/providers/oidc-provider \ + --subject-token-type urn:ietf:params:oauth:token-type:id_token \ + --credential-source-file gcp_cred \ + --workforce-pool-user-project antrea\ + --project antrea\ + --output-file gcp_cred_config.json + sudo gcloud auth login --cred-file=gcp_cred_config.json + sudo gcloud config set project antrea sudo ./ci/test-conformance-gke.sh --cluster-name antrea-gke-${{BUILD_NUMBER}} \ - --svc-account antrea-gcp@antrea.iam.gserviceaccount.com --gcloud-sdk-path ${{GCLOUD_SDK_PATH}} \ + --skip-iam-policy-binding --gcloud-sdk-path ${{GCLOUD_SDK_PATH}} \ --log-mode detail --setup-only triggers: - timed: H H */2 * * @@ -741,6 +753,86 @@ - email: notify-every-unstable-build: true recipients: projectantrea-dev@googlegroups.com + wrappers: + - credentials-binding: + - text: + credential-id: WORKFORCE_POOL # Jenkins secret that stores the cloud resource pool id + variable: WORKFORCE_POOL + - 'cloud-{name}-{test_name}-credential': + test_name: gke + node: antrea-credential + description: 'This is the cloud credential job for antrea gke tests.' + builders: + - shell: |- + #!/bin/bash + # Prevent exposing credentials in the console output by adding set +x + # This is to avoid other developers removing this set+x by accident + set +x + token_file="/var/lib/jenkins/ci_properties.txt" + if [ -e "$token_file" ]; then + modification_timestamp=$(stat -c %Y "$token_file") + current_timestamp=$(date +%s) + time_difference=$((current_timestamp - modification_timestamp)) + else + time_difference=14401 + fi + # The credentials are valid for 12 hours, to allow sufficient time for job execution, the reuse threshold is set to 4 hours. + threshold=14400 + if [ "$time_difference" -gt "$threshold" ]; then + echo "Generate the new Cloud Credential" + json_data=$(python3 get_access_using_api_client.py ${{CLOUD_CLIENT_ID}} ${{CLOUD_CLIENT_TOKEN}} PowerUser gcp ${{GKE_RESOURCE_ID}} 43200 prd) + json_data=$(echo $json_data | sed "s/'/\"/g" | sed 's/True/true/') + id_token=$(echo $json_data | jq -r '.credential.idToken') + echo "ID_TOKEN=$id_token" > ${{WORKSPACE}}/ci_properties.txt + cp ${{WORKSPACE}}/ci_properties.txt $token_file + else + echo "Reuse the unexpired Cloud Credential" + cp $token_file ${{WORKSPACE}}/ci_properties.txt + fi + triggers: + - timed: H H/6 * * * + publishers: + - trigger-parameterized-builds: + - project: + - cloud-{name}-renew-credential + current-parameters: true + property-file: 'ci_properties.txt' + - email: + notify-every-unstable-build: true + recipients: projectantrea-dev@googlegroups.com + wrappers: + - credentials-binding: + - text: + credential-id: CLOUD_CLIENT_ID # Jenkins secret that stores client id + variable: CLOUD_CLIENT_ID + - text: + credential-id: CLOUD_CLIENT_TOKEN # Jenkins secret that stores client secret token + variable: CLOUD_CLIENT_TOKEN + - text: + credential-id: GKE_RESOURCE_ID + variable: GKE_RESOURCE_ID + - 'cloud-{name}-{test_name}-credential': + test_name: renew + node: antrea-cloud + description: 'This is a periodic job to renew the credential on cloud node.' + builders: + - shell: |- + #!/bin/bash + # Prevent exposing credentials in the console output by adding set +x + # This is to avoid other developers removing this set+x by accident + set +x + cd /var/lib/jenkins + if [ -z "$ID_TOKEN" ]; then + echo "No Token ID Found" + exit 1 + else + echo "${{ID_TOKEN}}" > gcp_cred + fi + publishers: + - email: + notify-every-unstable-build: true + recipients: projectantrea-dev@googlegroups.com + triggers: [] wrappers: [] - 'cloud-{name}-{test_name}-cleanup': test_name: gke diff --git a/ci/test-conformance-gke.sh b/ci/test-conformance-gke.sh index 6f303626aa5..255fbcf3f92 100755 --- a/ci/test-conformance-gke.sh +++ b/ci/test-conformance-gke.sh @@ -29,6 +29,7 @@ MODE="report" RUN_ALL=true RUN_SETUP_ONLY=false RUN_CLEANUP_ONLY=false +SKIP_IAM_POLICY_BINDING=false TEST_SCRIPT_RC=0 KUBE_CONFORMANCE_IMAGE_VERSION=auto @@ -89,6 +90,10 @@ case $key in USER_EMAIL="$2" shift 2 ;; + --skip-iam-policy-binding) + SKIP_IAM_POLICY_BINDING=true + shift + ;; --gke-zone) GKE_ZONE="$2" shift 2 @@ -244,6 +249,8 @@ function deliver_antrea_to_gke() { elif [[ -n ${USER_EMAIL+x} ]]; then gcloud projects add-iam-policy-binding ${GKE_PROJECT} --member user:${USER_EMAIL} --role roles/container.admin kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user ${USER_EMAIL} + elif [[ "$SKIP_IAM_POLICY_BINDING" == true ]]; then + echo "Skipping the IAM Policy Binding for Cluster Management." else echo "Neither service account or user email info is set, cannot create cluster-admin-binding!" echo "Please refer to --help for more information."