Bump psycopg2 version to adopt libpq 16 (#15949)

* bump psycopg2 to include libpq 16 * fix unit tests post psycopg2 bump * update changelog * update agent requirements dependencies * update psycopg2-binary for pgbouncer * update changelog * add changelog entry to drop support for python2
DataDog · Oct 16, 2023 · 446874c · 446874c
1 parent 67f3e94
commit 446874c
Show file tree

Hide file tree

Showing 9 changed files with 50 additions and 22 deletions.
diff --git a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in
@@ -58,7 +58,7 @@ protobuf==3.17.3; python_version < '3.0'
 protobuf==3.20.2; python_version > '3.0'
 psutil==5.9.0
 psycopg-pool==3.1.7; python_version > '3.0'
-psycopg2-binary==2.8.6; sys_platform != 'darwin' or platform_machine != 'arm64'
+psycopg2-binary==2.9.8; python_version > '3.0'
 psycopg[binary]==3.1.10; python_version > '3.0'
 pyasn1==0.4.6
 pycryptodomex==3.10.1

diff --git a/pgbouncer/CHANGELOG.md b/pgbouncer/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+***Changed***:
+
+* Drop support for Python 2 ([#15949](https://github.com/DataDog/integrations-core/pull/15949))
+
+***Added***:
+
+* Upgrade `psycopg2-binary` to `v2.9.8` ([#15949](https://github.com/DataDog/integrations-core/pull/15949))
+
 ## 5.0.0 / 2023-08-10 / Agent 7.48.0
 
 ***Changed***:

diff --git a/pgbouncer/hatch.toml b/pgbouncer/hatch.toml
@@ -1,7 +1,7 @@
 [env.collectors.datadog-checks]
 
 [[envs.default.matrix]]
-python = ["2.7", "3.9"]
+python = ["3.9"]
 version = ["1.7", "1.8", "1.12"]
 
 [envs.default.env-vars]

diff --git a/pgbouncer/pyproject.toml b/pgbouncer/pyproject.toml
@@ -10,6 +10,7 @@ build-backend = "hatchling.build"
 name = "datadog-pgbouncer"
 description = "The PGBouncer check"
 readme = "README.md"
+requires-python = ">=3.8"
 keywords = [
     "datadog",
     "datadog agent",
@@ -40,7 +41,7 @@ license = "BSD-3-Clause"
 
 [project.optional-dependencies]
 deps = [
-    "psycopg2-binary==2.8.6; sys_platform != 'darwin' or platform_machine != 'arm64'",
+    "psycopg2-binary==2.9.8; python_version > '3.0'",
 ]
 
 [project.urls]

diff --git a/postgres/CHANGELOG.md b/postgres/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ***Added***:
 
+* Upgrade `psycopg2-binary` to `v2.9.8` ([#15949](https://github.com/DataDog/integrations-core/pull/15949))
 * Add support for reporting SQL obfuscation errors ([#15990](https://github.com/DataDog/integrations-core/pull/15990))
 
 ## 15.1.0 / 2023-10-06

diff --git a/postgres/pyproject.toml b/postgres/pyproject.toml
@@ -40,7 +40,7 @@ deps = [
     "azure-identity==1.14.0; python_version > '3.0'",
     "boto3==1.28.55; python_version > '3.0'",
     "cachetools==5.3.1; python_version > '3.0'",
-    "psycopg2-binary==2.8.6; sys_platform != 'darwin' or platform_machine != 'arm64'",
+    "psycopg2-binary==2.9.8; python_version > '3.0'",
     "semver==3.0.1; python_version > '3.0'",
 ]
 

diff --git a/postgres/tests/test_pg_integration.py b/postgres/tests/test_pg_integration.py
@@ -77,8 +77,13 @@ def test_common_metrics(aggregator, integration_check, pg_instance, is_aurora):
 
 def test_snapshot_xmin(aggregator, integration_check, pg_instance):
     with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn:
+        conn.set_session(autocommit=True)
         with conn.cursor() as cur:
-            cur.execute('select txid_snapshot_xmin(txid_current_snapshot());')
+            if float(POSTGRES_VERSION) >= 13.0:
+                query = 'select pg_snapshot_xmin(pg_current_snapshot());'
+            else:
+                query = 'select txid_snapshot_xmin(txid_current_snapshot());'
+            cur.execute(query)
             xmin = float(cur.fetchall()[0][0])
     check = integration_check(pg_instance)
     check.check(pg_instance)
@@ -92,13 +97,16 @@ def test_snapshot_xmin(aggregator, integration_check, pg_instance):
         conn.set_session(autocommit=True)
         with conn.cursor() as cur:
             # Force increases of txid
-            cur.execute('select txid_current();')
-            cur.execute('select txid_current();')
+            if float(POSTGRES_VERSION) >= 13.0:
+                query = 'select pg_current_xact_id();'
+            else:
+                query = 'select txid_current();'
+            cur.execute(query)
 
     check = integration_check(pg_instance)
     check.check(pg_instance)
-    aggregator.assert_metric('postgresql.snapshot.xmin', value=xmin + 2, count=1, tags=expected_tags)
-    aggregator.assert_metric('postgresql.snapshot.xmax', value=xmin + 2, count=1, tags=expected_tags)
+    aggregator.assert_metric('postgresql.snapshot.xmin', value=xmin + 1, count=1, tags=expected_tags)
+    aggregator.assert_metric('postgresql.snapshot.xmax', value=xmin + 1, count=1, tags=expected_tags)
 
 
 def test_snapshot_xip(aggregator, integration_check, pg_instance):

diff --git a/postgres/tests/test_pg_replication.py b/postgres/tests/test_pg_replication.py
@@ -101,16 +101,18 @@ def test_conflicts_lock(aggregator, integration_check, pg_instance, pg_replica_i
     expected_tags = _get_expected_tags(check, pg_replica_instance2, db=DB_NAME)
 
     replica_con = _get_superconn(pg_replica_instance2)
+    replica_con.set_session(autocommit=False)
     replica_cur = replica_con.cursor()
     replica_cur.execute('BEGIN;')
     replica_cur.execute('select * from persons;')
-    replica_cur.fetchall()
 
-    with _get_superconn(pg_instance) as conn:
-        conn.autocommit = True
-        with conn.cursor() as cur:
-            cur.execute('update persons SET personid = 1 where personid = 1;')
-            cur.execute('vacuum full persons')
+    conn = _get_superconn(pg_instance)
+    conn.set_session(autocommit=True)
+    cur = conn.cursor()
+    cur.execute('update persons SET personid = 1 where personid = 1;')
+    cur.execute('vacuum full persons;')
+    time.sleep(0.2)
+
     _wait_for_value(
         pg_replica_instance2,
         lower_threshold=0,
@@ -120,23 +122,28 @@ def test_conflicts_lock(aggregator, integration_check, pg_instance, pg_replica_i
     check.check(pg_replica_instance2)
     aggregator.assert_metric('postgresql.conflicts.lock', value=1, tags=expected_tags)
 
+    replica_con.close()
+    conn.close()
+
 
 @requires_over_10
 def test_conflicts_snapshot(aggregator, integration_check, pg_instance, pg_replica_instance2):
     check = integration_check(pg_replica_instance2)
     expected_tags = _get_expected_tags(check, pg_replica_instance2, db=DB_NAME)
 
     replica2_con = _get_superconn(pg_replica_instance2)
+    replica2_con.set_session(autocommit=False)
     replica2_cur = replica2_con.cursor()
     replica2_cur.execute('BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;')
     replica2_cur.execute('select * from persons;')
 
-    with _get_superconn(pg_instance) as conn:
-        conn.autocommit = True
-        with conn.cursor() as cur:
-            cur.execute('update persons SET personid = 1 where personid = 1;')
-            time.sleep(0.2)
-            cur.execute('vacuum verbose persons;')
+    conn = _get_superconn(pg_instance)
+    conn.set_session(autocommit=True)
+    cur = conn.cursor()
+    cur.execute('update persons SET personid = 1 where personid = 1;')
+    time.sleep(0.2)
+    cur.execute('vacuum verbose persons;')
+    time.sleep(0.2)
 
     _wait_for_value(
         pg_replica_instance2,
@@ -146,6 +153,9 @@ def test_conflicts_snapshot(aggregator, integration_check, pg_instance, pg_repli
     check.check(pg_replica_instance2)
     aggregator.assert_metric('postgresql.conflicts.snapshot', value=1, tags=expected_tags)
 
+    replica2_con.close()
+    conn.close()
+
 
 @pytest.mark.skip(reason="Failing on master")
 @requires_over_10

diff --git a/postgres/tests/utils.py b/postgres/tests/utils.py
@@ -52,7 +52,7 @@ def _wait_for_value(db_instance, lower_threshold, query):
             while value <= lower_threshold:
                 cur.execute(query)
                 value = cur.fetchall()[0][0]
-            time.sleep(0.1)
+                time.sleep(0.1)
 
 
 def run_one_check(check, db_instance):