Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding Lag-dependent implementation of Redis streams scaler #4592

Merged
merged 38 commits into from
Jun 21, 2023
Merged

Adding Lag-dependent implementation of Redis streams scaler #4592

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
94af8c4
Running e2e tests on new scaler
mikelam-us-aixplain May 1, 2023
80ca2e7
First hopefully successful test
mikelam-us-aixplain May 1, 2023
0cb0cb8
Scaler works now
mikelam-us-aixplain May 3, 2023
d3210ff
Pull request
mikelam-us-aixplain May 4, 2023
0686070
Cleaned up redis streams scaler to support xlag, xpending, and xlength
mikelam-us-aixplain May 15, 2023
acba7ef
Added tests and changed 'lag' to 'lagCount'
mikelam-us-aixplain May 18, 2023
f33c76b
Updated E2E tests to reflect lag -> lagCount
mikelam-us-aixplain May 18, 2023
eeea1b8
Updated protoc version and DeleteKubernetes call
mikelam-us-aixplain May 18, 2023
b6e1144
Added pending entries unit tests, corrected lagFactor error message, …
mikelam-us-aixplain May 30, 2023
fd45427
Removed files that no longer exist
mikelam-us-aixplain May 30, 2023
824c17f
Remove unneeded grpc files
mikelam-us-aixplain May 30, 2023
0b8e64e
Added unit tests and version check in scaler
mikelam-us-aixplain May 30, 2023
bb4d922
Fixing extraneous files
mikelam-us-aixplain May 30, 2023
bd97535
Unit tests working
mikelam-us-aixplain May 30, 2023
24bad85
Final saves
mikelam-us-aixplain May 30, 2023
f908ff6
int -> int64
mikelam-us-aixplain May 31, 2023
ac738c2
Fixed all int64 problems
mikelam-us-aixplain May 31, 2023
32b0b49
Moved test directories
mikelam-us-aixplain May 31, 2023
8f1c1ab
Use XINFO server to check version number
mikelam-us-aixplain Jun 1, 2023
3db35a1
Fixing style errors and docker repository
mikelam-us-aixplain Jun 1, 2023
2809d18
Fixed indentation error
mikelam-us-aixplain Jun 1, 2023
bbee9eb
Still fixing indentation erro
mikelam-us-aixplain Jun 1, 2023
e9b888c
Update redis_cluster_streams_pending_entries_test.go
mikelam-us Jun 1, 2023
eb982fb
Indentation should be fixed
mikelam-us-aixplain Jun 1, 2023
c700f85
Updated CHANGELOG and fixed Redis unit test
mikelam-us-aixplain Jun 1, 2023
a32a3ba
More flexible Redis version check
mikelam-us-aixplain Jun 1, 2023
f01dc41
Fixed version check
mikelam-us-aixplain Jun 1, 2023
66c7e51
Added activation value check andmade namespaces unique across tests
mikelam-us-aixplain Jun 6, 2023
8b76e88
Correcting stylistic errors
mikelam-us-aixplain Jun 6, 2023
4d27605
Fixed unnecessary leading newline
mikelam-us-aixplain Jun 6, 2023
6151763
Fixing formatting
mikelam-us-aixplain Jun 6, 2023
e0e771e
Streamlined e2e test
mikelam-us-aixplain Jun 7, 2023
0dcb105
Problem pushing commits
mikelam-us-aixplain Jun 7, 2023
ec2ce46
Fixed redis activation value bug and added standalone tests
mikelam-us-aixplain Jun 19, 2023
d6f442c
Updated in-line redis scaler documentation
mikelam-us-aixplain Jun 19, 2023
2120ecf
Merge branch 'main' into main
JorTurFer Jun 20, 2023
801b55d
Changed activationTargetLag -> activationLagCount
mikelam-us-aixplain Jun 20, 2023
d8b448a
Merge branch 'main' of https://github.com/mikelam-us/keda
mikelam-us-aixplain Jun 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 21 additions & 34 deletions tests/scalers/redis/redis_cluster_streams_lag/redis_cluster_streams_lag_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,35 +187,38 @@ func TestScaler(t *testing.T) {
data, templates := getTemplateData()

CreateKubernetesResources(t, kc, testNamespace, data, templates)
testScaleOut(t, kc, data)
testScaleIn(t, kc)
DeleteKubernetesResources(t, testNamespace, data, templates)
t.Log("--- testing activation ---")
testActivationValue(t, kc, data, 3)

t.Log("--- testing scale out with one more than activation ---")
testScaleOut(t, kc, data, 1, 1)

t.Log("--- testing scale out with many messages ---")
testScaleOut(t, kc, data, 100, maxReplicaCount)
JorTurFer marked this conversation as resolved.
Show resolved Hide resolved

CreateKubernetesResources(t, kc, testNamespace, activationData, templates)
testActivationValue(t, kc, activationData)
DeleteKubernetesResources(t, testNamespace, activationData, templates)
t.Log("--- testing scale in ---")
testScaleIn(t, kc, minReplicaCount)
DeleteKubernetesResources(t, testNamespace, data, templates)
// cleanup
redis.RemoveCluster(t, testName, redisNamespace)
}

func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) {
t.Log("--- testing scale out ---")
func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData, numMessages int, maxReplicas int) {
data.ItemsToWrite = numMessages
KubectlApplyWithTemplate(t, data, "insertJobTemplate", insertJobTemplate)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 3),
"replica count should be %d after 3 minutes", maxReplicaCount)
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicas, 60, 3),
"replica count should be %d after 3 minutes", maxReplicas)
}

func testScaleIn(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale in ---")

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3),
"replica count should be %d after 3 minutes", minReplicaCount)
func testScaleIn(t *testing.T, kc *kubernetes.Clientset, minReplicas int) {
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicas, 60, 3),
"replica count should be %d after 3 minutes", minReplicas)
}

func testActivationValue(t *testing.T, kc *kubernetes.Clientset, activationData templateData) {
t.Log("--- testing activation value ---")
KubectlApplyWithTemplate(t, activationData, "insertJobTemplate", insertJobTemplate)
func testActivationValue(t *testing.T, kc *kubernetes.Clientset, data templateData, numMessages int) {
data.ItemsToWrite = numMessages
KubectlApplyWithTemplate(t, data, "insertJobTemplate", insertJobTemplate)

time.Sleep(60 * time.Second)
AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 30)
Expand All @@ -237,22 +240,6 @@ var data = templateData{
ItemsToWrite: 100,
}

var activationData = templateData{
TestNamespace: testNamespace,
RedisNamespace: redisNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
MinReplicaCount: minReplicaCount,
MaxReplicaCount: maxReplicaCount,
TriggerAuthenticationName: triggerAuthenticationName,
SecretName: secretName,
JobName: jobName,
RedisPassword: redisPassword,
RedisPasswordBase64: base64.StdEncoding.EncodeToString([]byte(redisPassword)),
RedisHost: redisHost,
ItemsToWrite: 1,
}

func getTemplateData() (templateData, []Template) {
return data, []Template{
{Name: "secretTemplate", Config: secretTemplate},
Expand Down
57 changes: 21 additions & 36 deletions tests/scalers/redis/redis_sentinel_streams_lag/redis_sentinel_streams_lag_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,38 +201,39 @@ func TestScaler(t *testing.T) {
data, templates := getTemplateData()

CreateKubernetesResources(t, kc, testNamespace, data, templates)
testScaleOut(t, kc, data)
testScaleIn(t, kc)
DeleteKubernetesResources(t, testNamespace, data, templates)
t.Log("--- testing activation ---")
testActivationValue(t, kc, data, 3)

t.Log("--- testing scale out with one more than activation ---")
testScaleOut(t, kc, data, 1, 1)

t.Log("--- testing scale out with many messages ---")
testScaleOut(t, kc, data, 100, maxReplicaCount)
JorTurFer marked this conversation as resolved.
Show resolved Hide resolved

CreateKubernetesResources(t, kc, testNamespace, activationData, templates)
testActivationValue(t, kc, activationData)
DeleteKubernetesResources(t, testNamespace, activationData, templates)
t.Log("--- testing scale in ---")
testScaleIn(t, kc, minReplicaCount)
DeleteKubernetesResources(t, testNamespace, data, templates)

// cleanup
redis.RemoveCluster(t, testName, redisNamespace)
}

func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) {
t.Log("--- testing scale out ---")
func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData, numMessages int, maxReplicas int) {
data.ItemsToWrite = numMessages
KubectlApplyWithTemplate(t, data, "insertJobTemplate", insertJobTemplate)

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 3),
"replica count should be %d after 3 minutes", maxReplicaCount)
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicas, 60, 3),
"replica count should be %d after 3 minutes", maxReplicas)
}

func testScaleIn(t *testing.T, kc *kubernetes.Clientset) {
t.Log("--- testing scale in ---")

assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3),
"replica count should be %d after 3 minutes", minReplicaCount)

AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 30)
func testScaleIn(t *testing.T, kc *kubernetes.Clientset, minReplicas int) {
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicas, 60, 3),
"replica count should be %d after 3 minutes", minReplicas)
}

func testActivationValue(t *testing.T, kc *kubernetes.Clientset, activationData templateData) {
t.Log("--- testing activation value ---")
KubectlApplyWithTemplate(t, activationData, "insertJobTemplate", insertJobTemplate)
func testActivationValue(t *testing.T, kc *kubernetes.Clientset, data templateData, numMessages int) {
data.ItemsToWrite = numMessages
KubectlApplyWithTemplate(t, data, "insertJobTemplate", insertJobTemplate)

time.Sleep(60 * time.Second)
AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 30)
Expand All @@ -254,22 +255,6 @@ var data = templateData{
ItemsToWrite: 100,
}

var activationData = templateData{
TestNamespace: testNamespace,
RedisNamespace: redisNamespace,
DeploymentName: deploymentName,
ScaledObjectName: scaledObjectName,
MinReplicaCount: minReplicaCount,
MaxReplicaCount: maxReplicaCount,
TriggerAuthenticationName: triggerAuthenticationName,
SecretName: secretName,
JobName: jobName,
RedisPassword: redisPassword,
RedisPasswordBase64: base64.StdEncoding.EncodeToString([]byte(redisPassword)),
RedisHost: redisHost,
ItemsToWrite: 2,
}

func getTemplateData() (templateData, []Template) {
return data, []Template{
{Name: "secretTemplate", Config: secretTemplate},
Expand Down