add support python 3.12 and drop support python 3.9 (#447)

* add support python 3.12 and drop support python 3.9
* install setuptools because it is not in 3.12-slim image
* remove semgrep because it is not executable without cloud account anymore
---------

Co-authored-by: dtrai2 <95028228+dtrai2@users.noreply.github.com>

.devcontainer/devcontainer.json

@@ -9,7 +9,7 @@
 			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
 			// Append -bullseye or -buster to pin to an OS version.
 			// Use -bullseye variants on local on arm64/Apple Silicon.
-			"VARIANT": "3.11",
+			"VARIANT": "3.12",
 			// Options
 			"NODE_VERSION": "none"
 		}
@@ -40,4 +40,4 @@
 	// "postCreateCommand": "pip3 install --user -r requirements.txt",
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode"
-}
+}

.dockerignore

@@ -1,7 +1,6 @@
 .devcontainer
 .github
 .venv
-.semgrep_rules
 .vscode
 htmlcov
 logs

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -56,7 +56,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3
@@ -88,7 +88,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -117,7 +117,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -158,14 +158,10 @@ jobs:
       - name: Upload coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v2
 
-      - name: Check semgrep rules
-        if: steps.changed-files.outputs.all_changed_files
-        run: semgrep -c .semgrep_rules -c r/python --error --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}
-
   containerbuild:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     runs-on: ubuntu-latest
     steps:

.github/workflows/main.yml

@@ -11,7 +11,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -54,7 +54,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3
@@ -86,7 +86,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -127,7 +127,3 @@ jobs:
 
       - name: Upload coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v2
-
-      - name: Check semgrep rules
-        if: steps.changed-files.outputs.all_changed_files
-        run: semgrep -c .semgrep_rules -c r/python --error --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}

.github/workflows/publish-latest-dev-release-to-pypi.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Initialize Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: 3.10
 
       - name: Install dependencies
         run: |
@@ -39,7 +39,7 @@ jobs:
   containerbuild:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     runs-on: ubuntu-latest
     steps:

.github/workflows/publish-release-to-pypi.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Initialize Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: 3.10
           cache: "pip"
 
       - name: Install dependencies
@@ -55,7 +55,7 @@ jobs:
   containerbuild:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     runs-on: ubuntu-latest
     needs: publish-latest-release-to-pypi

.readthedocs.yaml

@@ -11,7 +11,7 @@ build:
   apt_packages:
     - "pandoc"
   tools:
-    python: "3.9"
+    python: "3.10"
     # You can also specify other tool versions:
     # nodejs: "16"
     # rust: "1.55"

.vscode/settings.json

@@ -1,10 +1,6 @@
 {
     "python.testing.pytestEnabled": true,
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestArgs": [
-        "tests",
-        "-v",
-    ],
     "python.analysis.importFormat": "absolute",
     "editor.formatOnSave": true,
     "editor.rulers": [
@@ -36,4 +32,4 @@
         "editor.defaultFormatter": "ms-python.black-formatter",
         "editor.formatOnSave": true
     },
-}
+}

CHANGELOG.md

@@ -24,6 +24,7 @@
   - removed `max_caching_days` config option
   - add `max_cached_pseudonymized_urls` config option which defaults to 1000
   - add lru caching for peudonymizatin of urls
+* add support for python 3.12 and remove support for python 3.9
 
 ### Bugfix
 

Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.9
+ARG PYTHON_VERSION=3.10
 
 FROM python:${PYTHON_VERSION}-bullseye as build
 ARG LOGPREP_VERSION=latest
@@ -11,7 +11,7 @@ WORKDIR /logprep
 RUN python -m venv /opt/venv
 # Make sure we use the virtualenv:
 ENV PATH="/opt/venv/bin:$PATH"
-RUN python -m pip install --upgrade pip wheel
+RUN python -m pip install --upgrade pip wheel setuptools
 
 RUN if [ "$LOGPREP_VERSION" = "dev" ]; then python setup.py sdist bdist_wheel && pip install ./dist/logprep-*.whl;\
     elif [ "$LOGPREP_VERSION" = "latest" ]; then pip install git+https://github.com/fkie-cad/Logprep.git@latest; \

README.md

@@ -236,7 +236,7 @@ Details about the rule language and how to write rules for the processors can be
 
 ### Installation
 
-Python should be present on the system, currently supported are the versions 3.9 - 3.11.
+Python should be present on the system, currently supported are the versions 3.10 - 3.12.
 
 To install Logprep you have following options:
 
@@ -283,16 +283,16 @@ Those can be executed via: `tox -e [name of the test environment]`.
 For Example:
 
 ```
-tox -e py39-all
+tox -e py310-all
 ```
 
 This runs all tests, calculates the test coverage and evaluates the code quality for the python 
-3.9 version.
+3.10 version.
 
 Multiple environments can be tested within one call: 
 
 ```
-tox -e py39-all -e py310-all -e py311-all
+tox -e py310-all -e py311-all -e py312-all
 ```
 
 If you want to run them in parallel attach the option `-p`.
@@ -308,21 +308,7 @@ tox -av
 In case the requirements change, the test environments must be rebuilt with the parameter `-r`:
 
 ```
-tox -e py39 -e py310 -e py311 -r
-```
-
-### Semgrep
-
-If you want to run semgrep rules run
-
-```
-pip install semgrep
-```
-
-Afterwards you can just call the tox environment with for example 
-
-```
-tox -e py39-semgrep
+tox -e py310 -e py311 -e py312 -r
 ```
 
 ### Running Logprep
@@ -479,7 +465,7 @@ Building the documentation is done by executing the following command from withi
 the project root directory:
 
 ```
-tox -e py39-docs
+tox -e py310-docs
 ```
 
 A HTML documentation can be then found in `doc/_build/html/index.html`.

doc/source/getting_started.rst

@@ -5,7 +5,7 @@ Getting Started
 Installation
 ============
 
-Python should be present on the system. Currently, Python 3.9 - 3.11 are supported.
+Python should be present on the system. Currently, Python 3.10 - 3.12 are supported.
 To install Logprep you have following options:
 
 **1. Option:** Installation via PyPI:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 100
-target-version = ['py39', 'py310', 'py311']
+target-version = ['py310', 'py311', 'py312']
 
 [tool.isort]
 profile = "black"

requirements.in

@@ -11,7 +11,7 @@ hyperscan>=0.4.0; sys_platform == 'linux' and platform_machine == 'x86_64'
 jsonref
 luqum
 mysql-connector-python
-numpy>=1.24.1
+numpy>=1.26.0
 opensearch-py
 prometheus_client
 protobuf>=3.20.2