Skip to content

Commit

Permalink
refactor: Replace lib/pq with pgx in a PostgreSQL scaler (kedacore#4705)
Browse files Browse the repository at this point in the history
Signed-off-by: Alexander Narsudinov <a.narsudinov@gmail.com>
  • Loading branch information
a-narsudinov authored and SpiritZhou committed Jun 30, 2023
1 parent 740813e commit daaaff1
Show file tree
Hide file tree
Showing 205 changed files with 54,298 additions and 7,800 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
- **Pulsar Scaler**: Improve error messages for unsuccessful connections ([#4563](https://github.com/kedacore/keda/issues/4563))
- **Security**: Enable secret scanning in GitHub repo
- **RabbitMQ Scaler**: Add support for `unsafeSsl` in trigger metadata ([#4448](https://github.com/kedacore/keda/issues/4448))
- **PostgreSQL Scaler**: Replace `lib/pq` with `pgx` ([#4704](https://github.com/kedacore/keda/issues/4704))
- **Prometheus Metrics**: Add new metric with KEDA build info ([#4647](https://github.com/kedacore/keda/issues/4647))
- **Prometheus Scaler**: Add support for Google Managed Prometheus ([#4675](https://github.com/kedacore/keda/pull/4675))
- **Solace Scaler**: Add new `messageReceiveRateTarget` metric to Solace Scaler ([#4665](https://github.com/kedacore/keda/issues/4665))
Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ require (
github.com/hashicorp/vault/api v1.9.2
github.com/imdario/mergo v0.3.16
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/jackc/pgx/v5 v5.4.0
github.com/joho/godotenv v1.5.1
github.com/lib/pq v1.10.9
github.com/microsoft/ApplicationInsights-Go v0.4.4
github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5
github.com/mitchellh/hashstructure v1.1.0
Expand Down Expand Up @@ -212,6 +212,8 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,12 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.0 h1:BSr+GCm4N6QcgIwv0DyTFHK9ugfEFF9DzSbbzxOiXU0=
github.com/jackc/pgx/v5 v5.4.0/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
Expand Down Expand Up @@ -525,8 +531,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
Expand Down
4 changes: 2 additions & 2 deletions pkg/scalers/postgresql_scaler.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"github.com/go-logr/logr"
// PostreSQL drive required for this scaler
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
v2 "k8s.io/api/autoscaling/v2"
"k8s.io/metrics/pkg/apis/external_metrics"

Expand Down Expand Up @@ -146,7 +146,7 @@ func parsePostgreSQLMetadata(config *ScalerConfig) (*postgreSQLMetadata, error)
}

func getConnection(meta *postgreSQLMetadata, logger logr.Logger) (*sql.DB, error) {
db, err := sql.Open("postgres", meta.connection)
db, err := sql.Open("pgx", meta.connection)
if err != nil {
logger.Error(err, fmt.Sprintf("Found error opening postgreSQL: %s", err))
return nil, err
Expand Down
2 changes: 2 additions & 0 deletions pkg/scalers/postgresql_scaler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ var testPostgreSQLConnectionstring = []postgreSQLConnectionStringTestData{
{metadata: map[string]string{"query": "test_query", "targetQueryValue": "5"}, authParam: map[string]string{"connection": "test_connection_from_auth"}, connectionString: "test_connection_from_auth"},
// from meta
{metadata: map[string]string{"query": "test_query", "targetQueryValue": "5", "host": "localhost", "port": "1234", "dbName": "testDb", "userName": "user", "sslmode": "required"}, connectionString: "host=localhost port=1234 user=user dbname=testDb sslmode=required password="},
// from meta, multiple hosts
{metadata: map[string]string{"query": "test_query", "targetQueryValue": "5", "host": "host1,host2", "port": "1234", "dbName": "testDb", "userName": "user", "sslmode": "required"}, connectionString: "host=host1,host2 port=1234 user=user dbname=testDb sslmode=required password="},
}

func TestPosgresSQLConnectionStringGeneration(t *testing.T) {
Expand Down
219 changes: 219 additions & 0 deletions tests/scalers/postgresql/helper/postgresql_helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
//go:build e2e
// +build e2e

package helper

import (
"fmt"
"strings"
)

const (
PostgresqlImage = "postgres:10.5"
)

const (
DeploymentTemplate = `apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: postgresql-update-worker
name: {{.DeploymentName}}
namespace: {{.TestNamespace}}
spec:
replicas: 0
selector:
matchLabels:
app: postgresql-update-worker
template:
metadata:
labels:
app: postgresql-update-worker
spec:
containers:
- image: ghcr.io/kedacore/tests-postgresql
imagePullPolicy: Always
name: postgresql-processor-test
command:
- /app
- update
env:
- name: TASK_INSTANCES_COUNT
value: "6000"
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: {{.SecretName}}
key: postgresql_conn_str
`
ScaledObjectTemplate = `apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{.ScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
scaleTargetRef:
name: {{.DeploymentName}}
pollingInterval: 5
cooldownPeriod: 10
minReplicaCount: {{.MinReplicaCount}}
maxReplicaCount: {{.MaxReplicaCount}}
triggers:
- type: postgresql
metadata:
targetQueryValue: "4"
activationTargetQueryValue: "5"
query: "SELECT CEIL(COUNT(*) / 5) FROM task_instance WHERE state='running' OR state='queued'"
authenticationRef:
name: {{.TriggerAuthenticationName}}
`
LowLevelRecordsJobTemplate = `apiVersion: batch/v1
kind: Job
metadata:
labels:
app: postgresql-insert-low-level-job
name: postgresql-insert-low-level-job
namespace: {{.TestNamespace}}
spec:
template:
metadata:
labels:
app: postgresql-insert-low-level-job
spec:
containers:
- image: ghcr.io/kedacore/tests-postgresql
imagePullPolicy: Always
name: postgresql-processor-test
command:
- /app
- insert
env:
- name: TASK_INSTANCES_COUNT
value: "20"
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: {{.SecretName}}
key: postgresql_conn_str
restartPolicy: Never
backoffLimit: 4
`
InsertRecordsJobTemplate = `apiVersion: batch/v1
kind: Job
metadata:
labels:
app: postgresql-insert-job
name: postgresql-insert-job
namespace: {{.TestNamespace}}
spec:
template:
metadata:
labels:
app: postgresql-insert-job
spec:
containers:
- image: ghcr.io/kedacore/tests-postgresql
imagePullPolicy: Always
name: postgresql-processor-test
command:
- /app
- insert
env:
- name: TASK_INSTANCES_COUNT
value: "10000"
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: {{.SecretName}}
key: postgresql_conn_str
restartPolicy: Never
backoffLimit: 4
`
TriggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: {{.TriggerAuthenticationName}}
namespace: {{.TestNamespace}}
spec:
secretTargetRef:
- parameter: connection
name: {{.SecretName}}
key: {{.SecretKey}}
`
PostgreSQLStatefulSetTemplate = `apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: {{.PostgreSQLStatefulSetName}}
name: {{.PostgreSQLStatefulSetName}}
namespace: {{.TestNamespace}}
spec:
replicas: 1
serviceName: {{.PostgreSQLStatefulSetName}}
selector:
matchLabels:
app: {{.PostgreSQLStatefulSetName}}
template:
metadata:
labels:
app: {{.PostgreSQLStatefulSetName}}
spec:
containers:
- name: postgresql
image: {{.PostgreSQLImage}}
env:
- name: POSTGRES_USER
value: {{.PostgreSQLUsername}}
- name: POSTGRES_PASSWORD
value: {{.PostgreSQLPassword}}
- name: POSTGRES_DB
value: {{.PostgreSQLDatabase}}
ports:
- name: postgresql
protocol: TCP
containerPort: 5432
readinessProbe:
tcpSocket:
port: 5432
initialDelaySeconds: 5
timeoutSeconds: 5
livenessProbe:
tcpSocket:
port: 5432
initialDelaySeconds: 5
timeoutSeconds: 5
volumeMounts:
- name: init-scripts
mountPath: /docker-entrypoint-initdb.d
volumes:
- name: init-scripts
configMap:
name: master-slave-config
optional: true
`
PostgreSQLServiceTemplate = `apiVersion: v1
kind: Service
metadata:
labels:
app: {{.PostgreSQLStatefulSetName}}
name: {{.PostgreSQLStatefulSetName}}
namespace: {{.TestNamespace}}
spec:
ports:
- port: 5432
protocol: TCP
targetPort: 5432
selector:
app: {{.PostgreSQLStatefulSetName}}
type: ClusterIP
`
)

func GetConnectionString(username string, password string, svcNames []string, namespace string, database string) string {
svcFQDNs := make([]string, 0, len(svcNames))
for _, svcName := range svcNames {
svcFQDNs = append(svcFQDNs, fmt.Sprintf("%s.%s.svc.cluster.local", svcName, namespace))
}
fqdns := strings.Join(svcFQDNs, ",")
return fmt.Sprintf("postgresql://%s:%s@%s:5432/%s?sslmode=disable", username, password, fqdns, database)
}
Loading

0 comments on commit daaaff1

Please sign in to comment.