Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Python3.12 support #1141

Merged
merged 11 commits into from
Jan 10, 2024
1 change: 1 addition & 0 deletions docsrc/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
'pytest-cov',
'pytest-timeout',
'redis',
'quantiphy',
'requests',
'rich',
'semver',
Expand Down
18 changes: 8 additions & 10 deletions src/analysis/plugin/compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,17 @@ def shutdown(self):

def yara_match_to_dict(match: yara.Match) -> dict:
"""Converts a ``yara.Match`` to the format that :py:class:`analysis.YaraPluginBase` would return."""
# FIXME (yara): Use this when we upgrade to yara-python 4.3.0
# for string_match in match.strings:
# for string_instance in string_match.instances:

strings = [(offset, identifier, data.hex()) for offset, identifier, data in match.strings]
# see YARA docs: https://yara.readthedocs.io/en/latest/yarapython.html#yara.StringMatchInstance
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unnecessary comment?

Copy link
Collaborator Author

@jstucke jstucke Jan 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I think it is useful since you can't easily look at the YARA source (because it is a compiled C library)

strings = [
(string_instance.offset, string_match.identifier, string_instance.matched_data.decode(errors='replace'))
for string_match in match.strings # type: yara.StringMatch
for string_instance in string_match.instances # type: yara.StringMatchInstance
]

return {
'meta': {
# Optional
'date': match.meta.get('date'),
# Optional
'author': match.meta.get('author'),
'description': match.meta['description'],
key: match.meta.get(key)
for key in ('open_source', 'software_name', 'website', 'date', 'author', 'description')
},
'rule': match.rule,
'strings': strings,
Expand Down
6 changes: 4 additions & 2 deletions src/helperFunctions/web_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from common_helper_files import get_binary_from_file
from matplotlib import cm, colors
from passlib.context import CryptContext
from si_prefix import si_format
from quantiphy import Quantity

from helperFunctions.fileSystem import get_template_dir

Expand Down Expand Up @@ -113,7 +113,9 @@ def cap_length_of_element(hid_element: str, maximum: int = 55) -> str:


def _format_si_prefix(number: float, unit: str) -> str:
return f'{si_format(number, precision=2)}{unit}'
quantity = Quantity(number, unit)
quantity.set_prefs(map_sf={'u': 'µ'})
return quantity.render(prec=2)


def format_time(seconds: float) -> str:
Expand Down
2 changes: 1 addition & 1 deletion src/install/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ def _install_plugins(distribution, skip_docker, only_docker=False):


def _install_yara():
yara_version = 'v4.2.3' # must be the same version as `yara-python` in `install/requirements_common.txt`
yara_version = 'v4.4.0' # must be the same version as `yara-python` in `install/requirements_common.txt`

yara_process = subprocess.run('yara --version', shell=True, stdout=PIPE, stderr=STDOUT, text=True)
if yara_process.returncode == 0 and yara_process.stdout.strip() == yara_version.strip('v'):
Expand Down
4 changes: 2 additions & 2 deletions src/install/requirements_backend.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
cryptography==41.0.6
docker==6.0.1
docker~=6.1.3
MarkupSafe==2.1.1
networkx==2.6.3
Pillow==10.0.1
Expand All @@ -13,4 +13,4 @@ flask==2.3.3
git+https://github.com/fkie-cad/common_helper_yara.git

# For plugin definition
semver==3.0.1
semver==3.0.2
6 changes: 2 additions & 4 deletions src/install/requirements_common.txt
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
# General build stuff
testresources==2.0.1

# FixMe: remove pinned version as soon as installation of ssdeep works again
setuptools<66

# General python dependencies
appdirs==1.4.4
flaky==3.7.0
Expand All @@ -22,7 +19,8 @@ rich==12.6.0
sqlalchemy==2.0.15
ssdeep==3.4
xmltodict==0.13.0
yara-python==4.2.3
# FixMe: pin to 4.4.x as soon as it releases
git+https://github.com/VirusTotal/yara-python@d0921c0

# Config validation
pydantic==2.1.1
Expand Down
10 changes: 5 additions & 5 deletions src/install/requirements_frontend.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ flask==2.3.3
flask_restx==1.1.0
flask_sqlalchemy==3.0.5
itsdangerous==2.1.2
matplotlib==3.5.3
matplotlib==3.7.3
more_itertools==9.0.0
prompt_toolkit==3.0.32
prompt_toolkit==3.0.41
python-dateutil==2.8.2
si-prefix==1.2.2
uwsgi==2.0.22
quantiphy==2.19
uwsgi==2.0.23
virtualenv

# must be below dependent packages (flask, flask-login, flask-restx)
Expand All @@ -23,4 +23,4 @@ werkzeug==2.3.8
bleach==5.0.1

# Figuring out if the analysis is outdated
semver==3.0.1
semver==3.0.2
2 changes: 1 addition & 1 deletion src/plugins/analysis/binwalk/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
capstone==4.0.2
cstruct==4.0
matplotlib==3.5.3
matplotlib==3.7.3
1 change: 0 additions & 1 deletion src/plugins/analysis/cve_lookup/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
pyxdameraulevenshtein==1.7.1
retry==0.9.2
4 changes: 3 additions & 1 deletion src/plugins/analysis/ip_and_uri_finder/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
git+https://github.com/fkie-cad/common_analysis_ip_and_uri.git
geoip2==4.6.0
geoip2==4.7.0
# dependency of geoip2 for python >= 3.12
aiohttp~=3.9.0
52 changes: 52 additions & 0 deletions src/test/unit/analysis/test_addons_yara.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
from io import FileIO
from pathlib import Path

import yara

from analysis.YaraPluginBase import YaraBasePlugin
from analysis.plugin.addons import Yara
from analysis.plugin.compat import yara_match_to_dict
from helperFunctions.fileSystem import get_src_dir
from test.common_helper import create_test_file_object

signature_file = str(Path(get_src_dir()) / 'test/unit/analysis/test.yara')
test_target = str(Path(get_src_dir()) / 'test/data/files/get_files_test/testfile1')

EXPECTED_RESULT = {
'matches': True,
'meta': {
'description': 'Generic Software',
'open_source': False,
'software_name': 'Test Software',
'website': 'http://www.fkie.fraunhofer.de',
},
'rule': 'testRule',
'strings': [(0, '$a', 'test'), (22, '$a', 'Test')],
}


class MockYaraPlugin(YaraBasePlugin):
def __init__(self):
self.signature_path = signature_file
self.NAME = 'test_plugin'


class MockYaraAddonPlugin(Yara):
def __init__(self):
self._rules = yara.compile(signature_file)


def test_output_is_compatible():
fo = create_test_file_object(test_target)
plugin = MockYaraPlugin()
plugin.process_object(fo)
assert fo.processed_analysis['test_plugin']['testRule'] == EXPECTED_RESULT

yara_addon_plugin = MockYaraAddonPlugin()
file = FileIO(test_target)
yara_matches = yara_addon_plugin.match(file)
assert all(isinstance(m, yara.Match) for m in yara_matches)
converted_match = yara_match_to_dict(yara_matches[0])
assert converted_match['strings'] == EXPECTED_RESULT['strings']
for key, value in EXPECTED_RESULT['meta'].items():
assert converted_match['meta'][key] == value
10 changes: 5 additions & 5 deletions src/test/unit/helperFunctions/test_web_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ def test_cap_length_of_element_short():
@pytest.mark.parametrize(
('number', 'unit', 'expected_output'),
[
(1, 'm', '1.00 m'),
(0.034, 'g', '34.00 mg'),
(0.0000123456789, 's', '12.35 µs'),
(1, 'm', '1 m'),
(0.034, 'g', '34 mg'),
(0.0000123456789, 's', '12.3 µs'),
(1234.5, 'm', '1.23 km'),
],
)
Expand All @@ -87,8 +87,8 @@ def test_format_si_prefix(number, unit, expected_output):
@pytest.mark.parametrize(
('seconds', 'expected_output'),
[
(2, '2.00 s'),
(0.2, '200.00 ms'),
(2, '2 s'),
(0.2, '200 ms'),
(120, '0:02:00'),
(100000, '1 day, 3:46:40'),
],
Expand Down
Loading