Merge branch '2.9' into backport-9930-to-2.9

ddtrace/appsec/_ddwaf/ddwaf_types.py

@@ -181,7 +181,7 @@ def create_without_limits(cls, struct: DDWafRulesType) -> "ddwaf_object":
     def struct(self) -> DDWafRulesType:
         """Generate a python structure from ddwaf_object"""
         if self.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_STRING:
-            return self.value.stringValue.decode("UTF-8", errors="ignore")
+            return self.value.stringValue[: self.nbEntries].decode("UTF-8", errors="ignore")
         if self.type == DDWAF_OBJ_TYPE.DDWAF_OBJ_MAP:
             return {
                 self.value.array[i].parameterName.decode("UTF-8", errors="ignore"): self.value.array[i].struct
@@ -211,7 +211,7 @@ def __repr__(self):
 
 class ddwaf_value(ctypes.Union):
     _fields_ = [
-        ("stringValue", ctypes.c_char_p),
+        ("stringValue", ctypes.POINTER(ctypes.c_char)),
         ("uintValue", ctypes.c_ulonglong),
         ("intValue", ctypes.c_longlong),
         ("array", ddwaf_object_p),

ddtrace/internal/ci_visibility/recorder.py

@@ -3,6 +3,7 @@
 import json
 import os
 from pathlib import Path
+import re
 import socket
 from typing import TYPE_CHECKING  # noqa:F401
 from typing import NamedTuple  # noqa:F401
@@ -102,12 +103,17 @@ class CIVisibilityAuthenticationException(Exception):
     pass
 
 
-def _extract_repository_name_from_url(repository_url):
-    # type: (str) -> str
+def _extract_repository_name_from_url(repository_url: str) -> str:
+    _REPO_NAME_REGEX = r".*/(?P<repo_name>.*?)(\.git)?$"
+
     try:
-        return parse.urlparse(repository_url).path.rstrip(".git").rpartition("/")[-1]
+        url_path = parse.urlparse(repository_url).path
+        matches = re.match(_REPO_NAME_REGEX, url_path, flags=re.IGNORECASE)
+        if matches:
+            return matches.group("repo_name")
+        log.warning("Cannot extract repository name from unexpected URL path: %s", url_path)
+        return repository_url
     except ValueError:
-        # In case of parsing error, default to repository url
         log.warning("Repository name cannot be parsed from repository_url: %s", repository_url)
         return repository_url
 
@@ -548,6 +554,12 @@ def _fetch_tests_to_skip(self, skipping_mode: str):
                 self._test_suites_to_skip = []
                 self._tests_to_skip = defaultdict(list)
 
+        except Exception:
+            log.warning("Error retrieving skippable test data, no tests will be skipped", exc_info=True)
+            error_type = ERROR_TYPES.UNKNOWN
+            self._test_suites_to_skip = []
+            self._tests_to_skip = defaultdict(list)
+
         finally:
             record_itr_skippable_request(
                 sw.elapsed() * 1000,

lib-injection/docker-compose.yml

@@ -30,6 +30,7 @@ services:
     environment:
       - PYTHONPATH=/datadog-lib
       - DD_TRACE_AGENT_URL=http://testagent:8126
+      - DD_TELEMETRY_FORWARDER_PATH=
     volumes:
       - ${TEMP_DIR:-/tmp/ddtrace_test}:/datadog-lib
 
@@ -45,5 +46,6 @@ services:
       - PYTHONPATH=/datadog-lib
       - DD_TRACE_AGENT_URL=http://testagent:8126
       - DD_TRACE_DEBUG=1
+      - DD_TELEMETRY_FORWARDER_PATH=
     volumes:
       - ${TEMP_DIR:-/tmp/ddtrace_test}:/datadog-lib

lib-injection/min_compatible_versions.csv

@@ -0,0 +1,188 @@
+This file was generated by scripts/min_compatible_versions.py
+pkg_name,min_version
+Flask-Cache,~=0.13.1
+Jinja2,~=2.11.0
+SQLAlchemy,==2.0.22
+WebTest,0
+Werkzeug,<1.0
+ai21,0
+aiobotocore,~=1.4.2
+aiofiles,0
+aiohttp,~=3.7
+aiohttp_jinja2,~=1.5.0
+aiomysql,~=0.1.0
+aiopg,~=0.16.0
+aiosqlite,0
+algoliasearch,~=2.5
+anyio,>=3.4.0
+aredis,0
+asgiref,~=3.0
+astunparse,0
+async_generator,~=1.10
+asyncpg,~=0.22.0
+asynctest,==0.13.0
+attrs,>=20
+austin-python,~=1.0
+blinker,0
+boto3,0
+botocore,~=1.13
+bottle,>=0.12
+bytecode,0
+cassandra-driver,~=3.24.0
+cattrs,<23.1.1
+celery,~=4.4
+cfn-lint,~=0.53.1
+channels,~=3.0
+cherrypy,>=17
+click,==7.1.2
+cohere,==4.57
+confluent-kafka,~=1.9.2
+coverage,0
+cryptography,<39
+daphne,0
+databases,0
+datadog-lambda,>=4.66.0
+ddsketch,>=3.0.0
+django,>=2.2
+django-pylibmc,>=0.6
+django-q,0
+django-redis,>=4.5
+django_hosts,~=4.0
+djangorestframework,>=3.11
+docker,0
+dogpile.cache,~=0.9
+dramatiq,0
+elasticsearch,~=7.13.0
+elasticsearch1,~=1.10.0
+elasticsearch2,~=2.5.0
+elasticsearch5,~=5.5.0
+elasticsearch6,~=6.8.0
+elasticsearch7,~=7.13.0
+elasticsearch7[async],0
+elasticsearch8,~=8.0.1
+elasticsearch[async],0
+envier,==0.5.1
+exceptiongroup,0
+falcon,~=3.0
+fastapi,~=0.64.0
+flask,~=0.12.0
+flask-caching,~=1.10.0
+flask-login,~=0.6.2
+gevent,~=20.12.0
+git+https://github.com/gnufede/pytest-memray.git@24a3c0735db99eedf57fb36c573680f9bab7cd73,0
+googleapis-common-protos,0
+graphene,~=3.0.0
+graphql-core,~=3.2.0
+graphql-relay,0
+greenlet,~=1.0.0
+grpcio,~=1.34.0
+gunicorn,==20.0.4
+gunicorn[gevent],0
+httpretty,<1.1
+httpx,~=0.17.0
+huggingface-hub,0
+hypothesis,<6.45.1
+importlib-metadata,0
+importlib_metadata,<5.0
+itsdangerous,<2.0
+jinja2,~=2.11.0
+kombu,>=4.2.0
+langchain,==0.0.192
+langchain-aws,0
+langchain-community,==0.0.14
+langchain-core,==0.1.52
+langchain-openai,==0.1.6
+langchain-pinecone,==0.1.0
+langchain_experimental,==0.0.47
+langsmith,==0.1.58
+logbook,~=1.0.0
+loguru,~=0.4.0
+mako,~=1.1.0
+mariadb,~=1.0.0
+markupsafe,<2.0
+mock,0
+molten,>=1.0
+mongoengine,~=0.23
+more_itertools,<8.11.0
+moto,>=1.0
+moto[all],<5.0
+msgpack,~=1.0.0
+mysql-connector-python,==8.0.5
+mysqlclient,~=2.0
+numexpr,0
+openai,==0.26.5
+openai[datalib],==1.30.1
+"openai[embeddings,datalib]",==0.27.2
+opensearch-py,0
+opensearch-py[async],0
+opensearch-py[requests],~=1.1.0
+opentelemetry-api,>=1
+opentelemetry-instrumentation-flask,<=0.37b0
+opentracing,>=2.0.0
+peewee,0
+pillow,0
+pinecone-client,==2.2.4
+pony,0
+protobuf,>=3
+psutil,0
+psycopg,~=3.0.18
+psycopg2-binary,~=2.8.0
+py-cpuinfo,~=8.0.0
+pycryptodome,0
+pyfakefs,0
+pylibmc,~=1.6.2
+pymemcache,~=3.4.2
+pymongo,~=3.11
+pymysql,~=0.10
+pynamodb,~=5.0
+pyodbc,~=4.0.31
+pyramid,~=1.10
+pysqlite3-binary,0
+pytest,~=4.0
+pytest-aiohttp,0
+pytest-asyncio,==0.21.1
+pytest-bdd,>=4.0
+pytest-benchmark,>=3.1.0
+pytest-cov,==2.9.0
+pytest-django,==3.10.0
+pytest-mock,==2.0.0
+pytest-randomly,0
+pytest-sanic,~=1.6.2
+python-consul,>=1.1
+python-json-logger,==2.0.7
+python-memcached,0
+redis,~=2.0
+redis-py-cluster,>=2.0
+reno,0
+requests,~=2.20.0
+requests-mock,>=1.4
+responses,~=0.16.0
+rich,0
+rq,~=1.8.0
+ruamel.yaml,0
+sanic,~=20.12
+sanic-testing,~=0.8.3
+scikit-learn,==1.0.2
+simplejson,0
+six,==1.12.0
+snowflake-connector-python,~=2.3.0
+sqlalchemy,~=1.2.18
+starlette,~=0.14.0
+structlog,~=20.2.0
+tests/contrib/pyramid/pserve_app,0
+tiktoken,0
+tornado,~=4.5.0
+tortoise-orm,0
+typing-extensions,0
+typing_extensions,0
+urllib3,~=1.0
+uwsgi,0
+vcrpy,==4.2.1
+vertica-python,>=0.6.0
+websockets,<11.0
+webtest,0
+werkzeug,<1.0
+wheel,0
+xmltodict,>=0.12
+yaaredis,~=2.0.0
+yarl,~=1.0