Attempt 5000 pod scale test.

bnapolitan · Jun 19, 2020 · e17358c · e17358c
1 parent e9ea95d
commit e17358c
Show file tree

Hide file tree

Showing 3 changed files with 101 additions and 4 deletions.
diff --git a/deploy-5000-pods.yaml b/deploy-5000-pods.yaml
@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy-5000-pods
+spec:
+  replicas: 5000
+  selector:
+      matchLabels:
+        app: deploy-5000-pods
+  template:
+    metadata:
+      name: test-pod-5000
+      labels:
+        app: deploy-5000-pods
+        tier: backend
+        track: stable
+    spec:
+      containers:
+        - name: hello
+          image: "gcr.io/google-samples/hello-go-gke:1.0"
+          ports:
+            - name: http
+              containerPort: 80
+          imagePullPolicy: IfNotPresent
+      nodeSelector:
+        eks.amazonaws.com/nodegroup: multi-node
diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
@@ -12,6 +12,13 @@ function down-test-cluster() {
 }
 
 function up-test-cluster() {
+    MNGS=""
+    if [[ "$RUN_SCALE_TESTS" == true ]]; then
+        MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}, "three-nodes":{"name":"three-nodes","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["m5.xlarge"],"volume-size":40}, "single-node":{"name":"single-node","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":1,"asg-max-size":1,"asg-desired-capacity":1,"instance-types":["m5.16xlarge"],"volume-size":40}, "single-node":{"name":"single-node","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":100,"asg-max-size":100,"asg-desired-capacity":100,"instance-types":["m5.xlarge"],"volume-size":40}}'
+    else
+        MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}'
+    fi
+
     echo -n "Configuring cluster $CLUSTER_NAME"
     AWS_K8S_TESTER_EKS_NAME=$CLUSTER_NAME \
         AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \
@@ -25,7 +32,7 @@ function up-test-cluster() {
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_CREATE=$ROLE_CREATE \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_ARN=$ROLE_ARN \
-        AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}, "three-nodes":{"name":"three-nodes","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["m5.xlarge"],"volume-size":40}, "single-node":{"name":"single-node","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":1,"asg-max-size":1,"asg-desired-capacity":1,"instance-types":["m5.16xlarge"],"volume-size":40}}' \
+        AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS=$MNGS \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_FETCH_LOGS=true \
         AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \

diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh
@@ -101,20 +101,21 @@ function run_scale_test_730_pods() {
     do
         ITERATION_START=$SECONDS
         $KUBECTL_PATH scale -f deploy-730-pods.yaml --replicas=730
-        sleep 20
+        sleep 100
         while [[ ! $($KUBECTL_PATH get deploy | grep 730/730) && $((SECONDS - ITERATION_START)) -lt 200 ]]
         do
-            sleep 1
+            sleep 2
             echo "Scaling UP"
             echo $($KUBECTL_PATH get deploy)
         done
 
         SCALE_UP_DURATION_ARRAY+=( $((SECONDS - ITERATION_START)) )
         MIDPOINT_START=$SECONDS
         $KUBECTL_PATH scale -f deploy-730-pods.yaml --replicas=0
+        sleep 100
         while [[ $($KUBECTL_PATH get pods) && $((SECONDS - MIDPOINT_START)) -lt 200 ]]
         do
-            sleep 1
+            sleep 2
             echo "Scaling DOWN"
             echo $($KUBECTL_PATH get deploy)
         done
@@ -152,3 +153,66 @@ function run_scale_test_730_pods() {
 
     echo "TIMELINE: 730 Pod scale test took $DEPLOY_DURATION seconds."
 }
+
+function run_scale_test_5000_pods() {
+    echo "Running scale tests against cluster"
+    DEPLOY_START=$SECONDS
+
+    SCALE_UP_DURATION_ARRAY=()
+    SCALE_DOWN_DURATION_ARRAY=()
+    while [ ${#SCALE_UP_DURATION_ARRAY[@]} -lt 3 ]
+    do
+        ITERATION_START=$SECONDS
+        $KUBECTL_PATH scale -f deploy-5000-pods.yaml --replicas=5000
+        sleep 100
+        while [[ ! $($KUBECTL_PATH get deploy | grep 5000/5000) && $((SECONDS - ITERATION_START)) -lt 200 ]]
+        do
+            sleep 2
+            echo "Scaling UP"
+            echo $($KUBECTL_PATH get deploy)
+        done
+
+        SCALE_UP_DURATION_ARRAY+=( $((SECONDS - ITERATION_START)) )
+        MIDPOINT_START=$SECONDS
+        $KUBECTL_PATH scale -f deploy-5000-pods.yaml --replicas=0
+        sleep 100
+        while [[ $($KUBECTL_PATH get pods) && $((SECONDS - MIDPOINT_START)) -lt 200 ]]
+        do
+            sleep 2
+            echo "Scaling DOWN"
+            echo $($KUBECTL_PATH get deploy)
+        done
+        SCALE_DOWN_DURATION_ARRAY+=($((SECONDS - MIDPOINT_START)))
+    done
+
+    echo "Times to scale up:"
+    INDEX=0
+    while [ $INDEX -lt ${#SCALE_UP_DURATION_ARRAY[@]} ]
+    do
+        echo ${SCALE_UP_DURATION_ARRAY[$INDEX]}
+        INDEX=$((INDEX + 1))
+    done
+    echo ""
+    echo "Times to scale down:"
+    INDEX=0
+    while [ $INDEX -lt ${#SCALE_DOWN_DURATION_ARRAY[@]} ]
+    do
+        echo "${SCALE_DOWN_DURATION_ARRAY[$INDEX]} seconds"
+        INDEX=$((INDEX + 1))
+    done
+    echo ""
+    DEPLOY_DURATION=$((SECONDS - DEPLOY_START))
+
+    now="pod-5000-scale-test-data-$(date +"%m-%d-%Y-%T").csv"
+    echo $now
+
+    echo $(date +"%m-%d-%Y-%T") >> $now
+    echo $((SCALE_UP_DURATION_ARRAY[0])), $((SCALE_DOWN_DURATION_ARRAY[0])) >> $now
+    echo $((SCALE_UP_DURATION_ARRAY[1])), $((SCALE_DOWN_DURATION_ARRAY[1])) >> $now
+    echo $((SCALE_UP_DURATION_ARRAY[2])), $((SCALE_DOWN_DURATION_ARRAY[2])) >> $now
+
+    cat $now
+    aws s3 cp $now s3://cni-scale-test-data
+
+    echo "TIMELINE: 5000 Pod scale test took $DEPLOY_DURATION seconds."
+}