diff --git a/tests/load/README.md b/tests/load/README.md
index d742d232..69b70313 100644
--- a/tests/load/README.md
+++ b/tests/load/README.md
@@ -7,11 +7,12 @@ using a tool named [locust][1].
* [Autopush Load Test Class Diagram][2]
* [Autopush Load Test Spreadsheet][3]
+* [Autopush Load Test Artifacts][4]
## Contributing
-This project uses [Poetry][4] for dependency management. For environment setup it is
-recommended to use [pyenv][5] and [pyenv-virtualenv][6], as they work nicely with
+This project uses [Poetry][5] for dependency management. For environment setup it is
+recommended to use [pyenv][6] and [pyenv-virtualenv][7], as they work nicely with
Poetry.
Project dependencies are listed in the `pyproject.toml` file.
@@ -21,8 +22,8 @@ To install the dependencies execute:
poetry install
```
-Contributors to this project are expected to execute [isort][7], [black][8], [flake8][9]
-and [mypy][10] for import sorting, linting, style guide enforcement and static type
+Contributors to this project are expected to execute [isort][8], [black][9], [flake8][10]
+and [mypy][11] for import sorting, linting, style guide enforcement and static type
checking respectively. Configurations are set in the `pyproject.toml` and `.flake8`
files.
@@ -40,7 +41,7 @@ Follow the steps bellow to execute the load tests locally:
#### 1. Configure Environment Variables
-Environment variables, listed bellow or specified by [Locust][11], can be set in
+Environment variables, listed bellow or specified by [Locust][12], can be set in
`tests\load\docker-compose.yml`.
| Environment Variable | Node(s) | Description |
@@ -97,11 +98,23 @@ make load-clean
Follow the steps bellow to execute the distributed load tests on GCP:
+### Calibration
+
+Following the addition of new features, such as a Locust Task or Locust User, it may be necessary
+to re-establish the recommended parameters of a performance test.
+
+| Parameter | Description |
+|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [WAIT TIME][13] | - Changing this cadence will increase or decrease the number of channel subscriptions and notifications sent by an AutopushUser.
- This value can be set in the Locust UI. |
+| [TASK WEIGHT][14] | - Changing this weight impacts the probability of a task being chosen for execution.
- This value is hardcoded in the task decorators of the AutopushUser class. |
+| USERS PER WORKER | - This value should be set to the maximum number of users a Locust worker can support given CPU and memory constraints.
- This value is hardcoded in the AutopushLoadTestShape class. |
+| LOCUST WORKERS | - This value is derived by dividing the total number of users needed for the performance test by the 'USERS PER WORKER'.
- This value is hardcoded in the AutopushLoadTestShape and the setup_k8s.sh script. |
+
### Setup Environment
#### 1. Start a GCP Cloud Shell
-The load tests can be executed from the [contextual-services-test-eng cloud shell][12].
+The load tests can be executed from the [contextual-services-test-eng cloud shell][15].
#### 2. Configure the Bash Script
@@ -135,7 +148,7 @@ The load tests can be executed from the [contextual-services-test-eng cloud shel
* To apply new changes to an existing GCP Cluster, execute the `setup_k8s.sh` file and
select the **setup** option.
* This option will consider the local commit history, creating new containers and
- deploying them (see [Container Registry][13])
+ deploying them (see [Container Registry][16])
### Run Test Session
@@ -169,7 +182,7 @@ the load test will stop automatically.
* The number of responses with errors (non-200 response codes) should be `0`
* Locust reports Failures via the "autopush_failures.csv" file and the UI
(under the "Failures" tab or the "Charts" tab)
-* [Grafana][14] reports Failures via the "HTTP Response codes" chart and the
+* [Grafana][17] reports Failures via the "HTTP Response codes" chart and the
"HTTP 5xx error rate" chart
**Exceptions**
@@ -197,7 +210,7 @@ the load test will stop automatically.
```bash
kubectl get pods -o wide
```
- * Upload the files to the [ConServ][15] drive and record the links in the
+ * Upload the files to the [ConServ][4] drive and record the links in the
spreadsheet
### Clean-up Environment
@@ -213,15 +226,19 @@ Execute the `setup_k8s.sh` file and select the **delete** option
[1]: https://locust.io/
[2]: https://miro.com/app/board/uXjVMx-kx9Q=/
[3]: https://docs.google.com/spreadsheets/d/1-nF8zR98IiLZlwP0ISwOB8cWRs74tJGM1PpgDSWdayE/edit#gid=0
-[4]: https://python-poetry.org/docs/#installation
-[5]: https://github.com/pyenv/pyenv#installation
-[6]: https://github.com/pyenv/pyenv-virtualenv#installation
-[7]: https://pycqa.github.io/isort/
-[8]: https://black.readthedocs.io/en/stable/
-[9]: https://flake8.pycqa.org/en/latest/
-[10]: https://mypy-lang.org/
-[11]: https://docs.locust.io/en/stable/configuration.html#environment-variables
-[12]: https://console.cloud.google.com/home/dashboard?q=search&referrer=search&project=spheric-keel-331521&cloudshell=false
-[13]: https://console.cloud.google.com/compute/instances?project=spheric-keel-331521
-[14]: https://earthangel-b40313e5.influxcloud.net/d/do4mmwcVz/autopush-gcp?orgId=1&refresh=1m
-[15]: https://drive.google.com/drive/folders/13a-9DZXmeoJw4sLlo56tLwtnRXoKauHO
+[4]: https://drive.google.com/drive/folders/13a-9DZXmeoJw4sLlo56tLwtnRXoKauHO
+[5]: https://python-poetry.org/docs/#installation
+[6]: https://github.com/pyenv/pyenv#installation
+[7]: https://github.com/pyenv/pyenv-virtualenv#installation
+[8]: https://pycqa.github.io/isort/
+[9]: https://black.readthedocs.io/en/stable/
+[10]: https://flake8.pycqa.org/en/latest/
+[11]: https://mypy-lang.org/
+[12]: https://docs.locust.io/en/stable/configuration.html#environment-variables
+[13]: https://docs.locust.io/en/stable/writing-a-locustfile.html#wait-time
+[14]: https://docs.locust.io/en/stable/writing-a-locustfile.html#task-decorator
+[15]: https://console.cloud.google.com/home/dashboard?q=search&referrer=search&project=spheric-keel-331521&cloudshell=false
+[16]: https://console.cloud.google.com/compute/instances?project=spheric-keel-331521
+[17]: https://earthangel-b40313e5.influxcloud.net/d/do4mmwcVz/autopush-gcp?orgId=1&refresh=1m
+
+
diff --git a/tests/load/kubernetes-config/locust-master-controller.yml b/tests/load/kubernetes-config/locust-master-controller.yml
index 48fb7f3b..665e7103 100644
--- a/tests/load/kubernetes-config/locust-master-controller.yml
+++ b/tests/load/kubernetes-config/locust-master-controller.yml
@@ -46,8 +46,8 @@ spec:
protocol: TCP
resources:
limits:
- cpu: 2
- memory: 2Gi
- requests:
cpu: 1
- memory: 1Gi
+ memory: 3Gi
+ requests:
+ cpu: 0.9
+ memory: 2Gi
diff --git a/tests/load/kubernetes-config/locust-worker-controller.yml b/tests/load/kubernetes-config/locust-worker-controller.yml
index d7af0dbd..a7b8a83d 100644
--- a/tests/load/kubernetes-config/locust-worker-controller.yml
+++ b/tests/load/kubernetes-config/locust-worker-controller.yml
@@ -34,8 +34,8 @@ spec:
value:
resources:
limits:
- cpu: 2
- memory: 2Gi
- requests:
cpu: 1
- memory: 1Gi
+ memory: 3Gi
+ requests:
+ cpu: 0.9
+ memory: 2Gi
diff --git a/tests/load/locustfiles/load.py b/tests/load/locustfiles/load.py
index 964d874f..1312598f 100644
--- a/tests/load/locustfiles/load.py
+++ b/tests/load/locustfiles/load.py
@@ -44,8 +44,8 @@ class AutopushLoadTestShape(LoadTestShape):
"""
MAX_RUN_TIME: int = 600 # 10 minutes
- WORKER_COUNT: int = 119 # Must match value defined in setup_k8s.sh
- USERS_PER_WORKER: int = 350 # Number of users supported on a worker running on a n1-standard-4
+ WORKER_COUNT: int = 150 # Must match value defined in setup_k8s.sh
+ USERS_PER_WORKER: int = 500 # Number of users supported on a worker running on a n1-standard-2
MAX_USERS: int = WORKER_COUNT * USERS_PER_WORKER
trend: QuadraticTrend
user_classes: list[Type[User]] = [AutopushUser]
diff --git a/tests/load/locustfiles/locustfile.py b/tests/load/locustfiles/locustfile.py
index 3b976ec9..a6a091d2 100644
--- a/tests/load/locustfiles/locustfile.py
+++ b/tests/load/locustfiles/locustfile.py
@@ -64,7 +64,7 @@ def _(parser: Any):
type=str,
env_var="AUTOPUSH_WAIT_TIME",
help="AutopushUser wait time between tasks",
- default="20, 25",
+ default="25, 30",
)
@@ -164,7 +164,7 @@ def on_ws_close(
if close_status_code or close_msg:
logger.info(f"WebSocket closed. status={close_status_code} msg={close_msg}")
- @task(weight=95)
+ @task(weight=98)
def send_notification(self):
"""Sends a notification to a registered endpoint while connected to Autopush."""
if not self.ws or not self.channels:
diff --git a/tests/load/setup_k8s.sh b/tests/load/setup_k8s.sh
index 4e92f734..fbd5113d 100644
--- a/tests/load/setup_k8s.sh
+++ b/tests/load/setup_k8s.sh
@@ -10,8 +10,8 @@ CLUSTER='autopush-locust-load-test'
TARGET='https://updates-autopush.stage.mozaws.net'
SCOPE='https://www.googleapis.com/auth/cloud-platform'
REGION='us-central1'
-WORKER_COUNT=119
-MACHINE_TYPE='n1-standard-4'
+WORKER_COUNT=150
+MACHINE_TYPE='n1-standard-2' # 2 CPUs + 7.50GB Memory
BOLD=$(tput bold)
NORM=$(tput sgr0)
DIRECTORY=$(pwd)
@@ -99,7 +99,8 @@ do
case $response in
create) #Setup Kubernetes Cluster
echo -e "==================== Creating the GKE cluster "
- $GCLOUD container clusters create $CLUSTER --region $REGION --scopes $SCOPE --enable-autoscaling --min-nodes "5" --max-nodes "30" --scopes=logging-write,storage-ro --addons HorizontalPodAutoscaling,HttpLoadBalancing --machine-type $MACHINE_TYPE
+ # The total-max-nodes = WORKER_COUNT + 1 (MASTER)
+ $GCLOUD container clusters create $CLUSTER --region $REGION --scopes $SCOPE --enable-autoscaling --total-min-nodes "1" --total-max-nodes "151" --scopes=logging-write,storage-ro --addons HorizontalPodAutoscaling,HttpLoadBalancing --machine-type $MACHINE_TYPE
SetupGksCluster
break
;;