Python3.12 support (#1141)

* replaced abandoned package 'si-prefix' with 'quantiphy'

* update matplotlib version and remove setuptools pin

* removed now obsolete dependency 'pyxdameraulevenshtein' of cve_lookup

* update remaining dependencies for python 3.12

* updated docker-py version to fix compatibility with newer requests

* fixed matplotlib version mismatch

* upgrade yara version

* plugin compat: updated for new yara version + added test

docsrc/conf.py

@@ -96,6 +96,7 @@
     'pytest-cov',
     'pytest-timeout',
     'redis',
+    'quantiphy',
     'requests',
     'rich',
     'semver',

src/analysis/plugin/compat.py

@@ -58,19 +58,17 @@ def shutdown(self):
 
 def yara_match_to_dict(match: yara.Match) -> dict:
     """Converts a ``yara.Match`` to the format that :py:class:`analysis.YaraPluginBase` would return."""
-    # FIXME (yara): Use this when we upgrade to yara-python 4.3.0
-    # for string_match in match.strings:
-    #    for string_instance in string_match.instances:
-
-    strings = [(offset, identifier, data.hex()) for offset, identifier, data in match.strings]
+    # see YARA docs: https://yara.readthedocs.io/en/latest/yarapython.html#yara.StringMatchInstance
+    strings = [
+        (string_instance.offset, string_match.identifier, string_instance.matched_data.decode(errors='replace'))
+        for string_match in match.strings  # type: yara.StringMatch
+        for string_instance in string_match.instances  # type: yara.StringMatchInstance
+    ]
 
     return {
         'meta': {
-            # Optional
-            'date': match.meta.get('date'),
-            # Optional
-            'author': match.meta.get('author'),
-            'description': match.meta['description'],
+            key: match.meta.get(key)
+            for key in ('open_source', 'software_name', 'website', 'date', 'author', 'description')
         },
         'rule': match.rule,
         'strings': strings,

src/helperFunctions/web_interface.py

@@ -7,7 +7,7 @@
 from common_helper_files import get_binary_from_file
 from matplotlib import cm, colors
 from passlib.context import CryptContext
-from si_prefix import si_format
+from quantiphy import Quantity
 
 from helperFunctions.fileSystem import get_template_dir
 
@@ -113,7 +113,9 @@ def cap_length_of_element(hid_element: str, maximum: int = 55) -> str:
 
 
 def _format_si_prefix(number: float, unit: str) -> str:
-    return f'{si_format(number, precision=2)}{unit}'
+    quantity = Quantity(number, unit)
+    quantity.set_prefs(map_sf={'u': 'µ'})
+    return quantity.render(prec=2)
 
 
 def format_time(seconds: float) -> str:

src/install/backend.py

@@ -120,7 +120,7 @@ def _install_plugins(distribution, skip_docker, only_docker=False):
 
 
 def _install_yara():
-    yara_version = 'v4.2.3'  # must be the same version as `yara-python` in `install/requirements_common.txt`
+    yara_version = 'v4.4.0'  # must be the same version as `yara-python` in `install/requirements_common.txt`
 
     yara_process = subprocess.run('yara --version', shell=True, stdout=PIPE, stderr=STDOUT, text=True)
     if yara_process.returncode == 0 and yara_process.stdout.strip() == yara_version.strip('v'):

src/install/requirements_backend.txt

@@ -1,5 +1,5 @@
 cryptography==41.0.6
-docker==6.0.1
+docker~=6.1.3
 MarkupSafe==2.1.1
 networkx==2.6.3
 Pillow==10.0.1
@@ -13,4 +13,4 @@ flask==2.3.3
 git+https://github.com/fkie-cad/common_helper_yara.git
 
 # For plugin definition
-semver==3.0.1
+semver==3.0.2

src/install/requirements_common.txt

@@ -1,9 +1,6 @@
 # General build stuff
 testresources==2.0.1
 
-# FixMe: remove pinned version as soon as installation of ssdeep works again
-setuptools<66
-
 # General python dependencies
 appdirs==1.4.4
 flaky==3.7.0
@@ -22,7 +19,8 @@ rich==12.6.0
 sqlalchemy==2.0.15
 ssdeep==3.4
 xmltodict==0.13.0
-yara-python==4.2.3
+# FixMe: pin to 4.4.x as soon as it releases
+git+https://github.com/VirusTotal/yara-python@d0921c0
 
 # Config validation
 pydantic==2.1.1

src/install/requirements_frontend.txt

@@ -8,12 +8,12 @@ flask==2.3.3
 flask_restx==1.1.0
 flask_sqlalchemy==3.0.5
 itsdangerous==2.1.2
-matplotlib==3.5.3
+matplotlib==3.7.3
 more_itertools==9.0.0
-prompt_toolkit==3.0.32
+prompt_toolkit==3.0.41
 python-dateutil==2.8.2
-si-prefix==1.2.2
-uwsgi==2.0.22
+quantiphy==2.19
+uwsgi==2.0.23
 virtualenv
 
 # must be below dependent packages (flask, flask-login, flask-restx)
@@ -23,4 +23,4 @@ werkzeug==2.3.8
 bleach==5.0.1
 
 # Figuring out if the analysis is outdated
-semver==3.0.1
+semver==3.0.2

src/plugins/analysis/binwalk/requirements.txt

@@ -1,3 +1,3 @@
 capstone==4.0.2
 cstruct==4.0
-matplotlib==3.5.3
+matplotlib==3.7.3

src/plugins/analysis/cve_lookup/requirements.txt

@@ -1,2 +1 @@
-pyxdameraulevenshtein==1.7.1
 retry==0.9.2

src/plugins/analysis/ip_and_uri_finder/requirements.txt

@@ -1,2 +1,4 @@
 git+https://github.com/fkie-cad/common_analysis_ip_and_uri.git
-geoip2==4.6.0
+geoip2==4.7.0
+# dependency of geoip2 for python >= 3.12
+aiohttp~=3.9.0

src/test/unit/analysis/test_addons_yara.py

@@ -0,0 +1,52 @@
+from io import FileIO
+from pathlib import Path
+
+import yara
+
+from analysis.YaraPluginBase import YaraBasePlugin
+from analysis.plugin.addons import Yara
+from analysis.plugin.compat import yara_match_to_dict
+from helperFunctions.fileSystem import get_src_dir
+from test.common_helper import create_test_file_object
+
+signature_file = str(Path(get_src_dir()) / 'test/unit/analysis/test.yara')
+test_target = str(Path(get_src_dir()) / 'test/data/files/get_files_test/testfile1')
+
+EXPECTED_RESULT = {
+    'matches': True,
+    'meta': {
+        'description': 'Generic Software',
+        'open_source': False,
+        'software_name': 'Test Software',
+        'website': 'http://www.fkie.fraunhofer.de',
+    },
+    'rule': 'testRule',
+    'strings': [(0, '$a', 'test'), (22, '$a', 'Test')],
+}
+
+
+class MockYaraPlugin(YaraBasePlugin):
+    def __init__(self):
+        self.signature_path = signature_file
+        self.NAME = 'test_plugin'
+
+
+class MockYaraAddonPlugin(Yara):
+    def __init__(self):
+        self._rules = yara.compile(signature_file)
+
+
+def test_output_is_compatible():
+    fo = create_test_file_object(test_target)
+    plugin = MockYaraPlugin()
+    plugin.process_object(fo)
+    assert fo.processed_analysis['test_plugin']['testRule'] == EXPECTED_RESULT
+
+    yara_addon_plugin = MockYaraAddonPlugin()
+    file = FileIO(test_target)
+    yara_matches = yara_addon_plugin.match(file)
+    assert all(isinstance(m, yara.Match) for m in yara_matches)
+    converted_match = yara_match_to_dict(yara_matches[0])
+    assert converted_match['strings'] == EXPECTED_RESULT['strings']
+    for key, value in EXPECTED_RESULT['meta'].items():
+        assert converted_match['meta'][key] == value

src/test/unit/helperFunctions/test_web_interface.py

@@ -74,9 +74,9 @@ def test_cap_length_of_element_short():
 @pytest.mark.parametrize(
     ('number', 'unit', 'expected_output'),
     [
-        (1, 'm', '1.00 m'),
-        (0.034, 'g', '34.00 mg'),
-        (0.0000123456789, 's', '12.35 µs'),
+        (1, 'm', '1 m'),
+        (0.034, 'g', '34 mg'),
+        (0.0000123456789, 's', '12.3 µs'),
         (1234.5, 'm', '1.23 km'),
     ],
 )
@@ -87,8 +87,8 @@ def test_format_si_prefix(number, unit, expected_output):
 @pytest.mark.parametrize(
     ('seconds', 'expected_output'),
     [
-        (2, '2.00 s'),
-        (0.2, '200.00 ms'),
+        (2, '2 s'),
+        (0.2, '200 ms'),
         (120, '0:02:00'),
         (100000, '1 day, 3:46:40'),
     ],