From fb2b9d6ae0556dd21dcf6b84fac43459a6e8d1c7 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Feb 2025 15:46:07 +0100 Subject: [PATCH 1/7] Split path manipulation and file type validation --- server/mergin/sync/public_api_controller.py | 18 ++- server/mergin/sync/utils.py | 125 ++++++++++++++++++ .../mergin/tests/test_project_controller.py | 72 +++++++++- 3 files changed, 210 insertions(+), 5 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index bf99b55b..4bea06a6 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -4,6 +4,7 @@ import binascii import functools +import io import json import mimetypes import os @@ -87,6 +88,8 @@ get_project_path, get_device_id, is_valid_path, + is_supported_type, + is_supported_extension, ) from .errors import StorageLimitHit from ..utils import format_time_delta @@ -813,7 +816,9 @@ def project_push(namespace, project_name): if not all(ele.path != item.path for ele in project.files): abort(400, f"File {item.path} has been already uploaded") if not is_valid_path(item.path): - abort(400, f"File {item.path} contains invalid characters.") + abort(400, f"File '{item.path}' contains invalid characters.") + if not is_supported_extension(item.path): + abort(400, f"Unsupported extension of '{item.path}' file.") # changes' files must be unique changes_files = [ @@ -1472,3 +1477,14 @@ def get_project_version(project_id: str, version: str): ).first_or_404() data = ProjectVersionSchema(exclude=["files"]).dump(pv) return data, 200 + + +def validate_file(stream, filename): + """Check file type (from its content) for unsupported types""" + buffer_for_mime_check = stream.read(2048) + if not is_supported_type(buffer_for_mime_check): + abort(400, f"Unsupported file type of '{filename}' file.") + # reset the stream position for normal reading + validated_stream = io.BytesIO(buffer_for_mime_check + stream.read()) + return validated_stream + diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index f844d6b6..c3d85452 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -21,6 +21,7 @@ is_valid_filepath, is_valid_filename, ) +import magic def generate_checksum(file, chunk_size=4096): @@ -368,3 +369,127 @@ def is_valid_path(filepath: str) -> bool: os.path.basename(filepath) ) # invalid characters in filename, reserved filenames ) + + +def is_supported_extension(filepath) -> bool: + """Check whether file's extension is supported.""" + ext = os.path.splitext(filepath)[1].lower() + return ext in ALLOWED_EXTENSIONS + + +ALLOWED_EXTENSIONS = { + # Geospatial + # Shapefile components + ".shp", + ".shx", + ".dbf", + ".prj", + ".cpg", + ".qix", + ".sbn", + ".sbx", + # Vector data formats + ".geojson", + ".kml", + ".kmz", + ".gpx", + ".dxf", + ".svg", + ".gpkg", + ".json", + # Raster data formats + ".tif", + ".tiff", + ".geotiff", + ".asc", + ".vrt", + ".grd", + ".img", + ".adf", + # Point cloud data formats + ".las", + ".laz", + ".ply", + ".xyz", + ".e57", + ".pcd", + # Database and container formats + ".mbtiles", + ".sqlite", + ".gpkg", + # Geospatial metadata and styles + ".sld", + ".qml", + ".lyr", + ".qgz", + ".qgs", + # Other specialized formats + ".hdf", + ".hdf5", + ".netcdf", + ".nc", + ".dem", + ".dt2", + ".dt0", + ".map", + ".tab", + ".mif", + ".mid", + # Images + ".jpg", + ".jpeg", + ".png", + ".bmp", + ".gif", + ".heic", + ".webp", + ".tif", + ".tiff", + # Text documents + ".pdf", + ".doc", + ".docx", + ".odt", + ".rtf", + ".txt", + # Others + ".zip", +} + +ALLOWED_MIME_TYPES = { + "application/x-shapefile", + "application/x-dbf", + "text/plain", + "application/octet-stream", + "application/geo+json", + "application/vnd.google-earth.kml+xml", + "application/vnd.google-earth.kmz", + "application/gpx+xml", + "application/geopackage+sqlite3", + "application/vnd.sqlite3", + "application/json", + "text/xml", + "text/html", + "application/vnd.mapbox-vector-tile", + "application/x-sqlite3", + "application/vnd.ogc.sld+xml", + "application/xml", + "application/x-qgis", + "application/x-hdf", + "application/x-hdf5", + "application/x-netcdf", + "application/pdf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.oasis.opendocument.text", + "application/rtf", + "text/plain", + "application/zip", +} + + +def is_supported_type(head) -> bool: + """Check whether the file mimetype is supported.""" + mime_type = magic.Magic(mime=True).from_buffer(head) + return mime_type.startswith("image/") or mime_type in ALLOWED_MIME_TYPES + diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 9d8671e8..f26946c9 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -2514,8 +2514,8 @@ def test_signals(client): push_finished_mock.assert_called_once() -def test_upload_validation(client): - """Test filepath and filename validation during file upload""" +def test_filepath_manipulation(client): + """Test filepath validation during file upload""" push_start_url = url_for( f"/v1.mergin_sync_public_api_controller_project_push", namespace=test_workspace_name, @@ -2530,7 +2530,7 @@ def test_upload_validation(client): "removed": [], } # Manipulate the path by prepending ../../ - manipulated_path = "../../image.png" + manipulated_path = "../../" + filename changes["added"][0]["path"] = manipulated_path # Block script upload in push_start because of the invalid path resp = client.post( @@ -2542,5 +2542,69 @@ def test_upload_validation(client): ) assert resp.status_code == 400 assert ( - resp.json["detail"] == f"File {manipulated_path} contains invalid characters." + resp.json["detail"] == f"File '{manipulated_path}' contains invalid characters." ) + + +def test_supported_file_upload(client): + """Test rejecting unsupported file based on extension and its mime type""" + push_start_url = url_for( + f"/v1.mergin_sync_public_api_controller_project_push", + namespace=test_workspace_name, + project_name=test_project, + ) + content = """#!/bin/bash + echo "Hello Mergin!" + """ + script_filename = "script.sh" + with open(os.path.join(TMP_DIR, script_filename), "w") as f: + f.write(content) + changes = { + "added": [file_info(TMP_DIR, script_filename, chunk_size=CHUNK_SIZE)], + "updated": [], + "removed": [], + } + # Block script upload during push_start because of the unsupported extension + resp = client.post( + push_start_url, + data=json.dumps( + {"version": "v1", "changes": changes}, cls=DateTimeEncoder + ).encode("utf-8"), + headers=json_headers, + ) + assert resp.status_code == 400 + assert resp.json["detail"] == f"Unsupported extension of '{script_filename}' file." + # Extension spoofing to trick the validator + spoof_name = "script.gpkg" + os.rename(os.path.join(TMP_DIR, script_filename), os.path.join(TMP_DIR, spoof_name)) + changes = { + "added": [file_info(TMP_DIR, spoof_name, chunk_size=CHUNK_SIZE)], + "updated": [], + "removed": [], + } + # File passes the extension check in push_start + resp = client.post( + push_start_url, + data=json.dumps( + {"version": "v1", "changes": changes}, cls=DateTimeEncoder + ).encode("utf-8"), + headers=json_headers, + ) + assert resp.status_code == 200 + upload = Upload.query.get(resp.json["transaction"]) + assert upload + # Unsupported file type is revealed in chunk_upload based on the mime type and upload is refused + for file in changes["added"]: + for chunk_id in file["chunks"]: + url = "/v1/project/push/chunk/{}/{}".format(upload.id, chunk_id) + with open(os.path.join(TMP_DIR, file["path"]), "rb") as f: + data = f.read(CHUNK_SIZE) + checksum = hashlib.sha1() + checksum.update(data) + resp = client.post( + url, data=data, headers={"Content-Type": "application/octet-stream"} + ) + assert resp.status_code == 400 + assert ( + resp.json["detail"] == f"Unsupported file type of '{spoof_name}' file." + ) From 6d2c22b5b70075106ccb3d06ce2da4780112ae8c Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 10 Feb 2025 10:09:32 +0100 Subject: [PATCH 2/7] Switch to blacklists --- server/mergin/sync/utils.py | 272 +++++++++++++++++++++--------------- 1 file changed, 160 insertions(+), 112 deletions(-) diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index c3d85452..ce7b0592 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -366,130 +366,178 @@ def is_valid_path(filepath: str) -> bool: not len(re.split(r"\.[/\\]", filepath)) > 1 # ./ or .\ and is_valid_filepath(filepath) # invalid characters in filepath, absolute path and is_valid_filename( - os.path.basename(filepath) - ) # invalid characters in filename, reserved filenames + os.path.basename(filepath) + ) # invalid characters in filename, reserved filenames ) def is_supported_extension(filepath) -> bool: """Check whether file's extension is supported.""" ext = os.path.splitext(filepath)[1].lower() - return ext in ALLOWED_EXTENSIONS - - -ALLOWED_EXTENSIONS = { - # Geospatial - # Shapefile components - ".shp", - ".shx", - ".dbf", - ".prj", - ".cpg", - ".qix", - ".sbn", - ".sbx", - # Vector data formats - ".geojson", - ".kml", - ".kmz", - ".gpx", - ".dxf", - ".svg", - ".gpkg", - ".json", - # Raster data formats - ".tif", - ".tiff", - ".geotiff", - ".asc", - ".vrt", - ".grd", - ".img", - ".adf", - # Point cloud data formats - ".las", - ".laz", - ".ply", - ".xyz", - ".e57", + return ext not in FORBIDDEN_EXTENSIONS + + +FORBIDDEN_EXTENSIONS = { + ".ade", + ".adp", + ".app", + ".appcontent-ms", + ".application", + ".appref-ms", + ".asp", + ".aspx", + ".asx", + ".bas", + ".bat", + ".bgi", + ".cab", + ".cdxml", + ".cer", + ".chm", + ".cmd", + ".cnt", + ".com", + ".cpl", + ".crt", + ".csh", + ".der", + ".diagcab", + ".dll", + ".drv", + ".exe", + ".fxp", + ".gadget", + ".grp", + ".hlp", + ".hpj", + ".hta", + ".htc", + ".htaccess", + ".htpasswd", + ".inf", + ".ins", + ".iso", + ".isp", + ".its", + ".jar", + ".jnlp", + ".js", + ".jse", + ".jsp", + ".ksh", + ".lnk", + ".mad", + ".maf", + ".mag", + ".mam", + ".maq", + ".mar", + ".mas", + ".mat", + ".mau", + ".mav", + ".maw", + ".mcf", + ".mda", + ".mdb", + ".mde", + ".mdt", + ".mdw", + ".mdz", + ".msc", + ".mht", + ".mhtml", + ".msh", + ".msh1", + ".msh2", + ".mshxml", + ".msh1xml", + ".msh2xml", + ".msi", + ".msp", + ".mst", + ".msu", + ".ops", + ".osd", ".pcd", - # Database and container formats - ".mbtiles", - ".sqlite", - ".gpkg", - # Geospatial metadata and styles - ".sld", - ".qml", - ".lyr", - ".qgz", - ".qgs", - # Other specialized formats - ".hdf", - ".hdf5", - ".netcdf", - ".nc", - ".dem", - ".dt2", - ".dt0", - ".map", - ".tab", - ".mif", - ".mid", - # Images - ".jpg", - ".jpeg", - ".png", - ".bmp", - ".gif", - ".heic", - ".webp", - ".tif", - ".tiff", - # Text documents - ".pdf", - ".doc", - ".docx", - ".odt", - ".rtf", - ".txt", - # Others - ".zip", + ".pif", + ".pl", + ".plg", + ".prf", + ".prg", + ".printerexport", + ".ps1", + ".ps1xml", + ".ps2", + ".ps2xml", + ".psc1", + ".psc2", + ".psd1", + ".psdm1", + ".pssc", + ".pst", + ".py", + ".pyc", + ".pyo", + ".pyw", + ".pyz", + ".pyzw", + ".reg", + ".scf", + ".scr", + ".sct", + ".settingcontent-ms", + ".sh", + ".shb", + ".shs", + ".sys", + ".theme", + ".tmp", + ".torrent", + ".url", + ".vb", + ".vbe", + ".vbp", + ".vbs", + ".vhd", + ".vhdx", + ".vsmacros", + ".vsw", + ".webpnp", + ".website", + ".ws", + ".wsb", + ".wsc", + ".wsf", + ".wsh", + ".xbap", + ".xll", + ".xnk", } -ALLOWED_MIME_TYPES = { - "application/x-shapefile", - "application/x-dbf", - "text/plain", - "application/octet-stream", - "application/geo+json", - "application/vnd.google-earth.kml+xml", - "application/vnd.google-earth.kmz", - "application/gpx+xml", - "application/geopackage+sqlite3", - "application/vnd.sqlite3", - "application/json", - "text/xml", - "text/html", - "application/vnd.mapbox-vector-tile", - "application/x-sqlite3", - "application/vnd.ogc.sld+xml", - "application/xml", - "application/x-qgis", - "application/x-hdf", - "application/x-hdf5", - "application/x-netcdf", - "application/pdf", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.oasis.opendocument.text", - "application/rtf", - "text/plain", - "application/zip", +FORBIDDEN_MIME_TYPES = { + "application/x-msdownload", + "application/x-sh", + "application/x-bat", + "application/x-msdos-program", + "application/x-dosexec", + "application/x-csh", + "application/x-perl", + "application/javascript", + "application/x-python-code", + "application/x-ruby", + "application/java-archive", + "application/vnd.ms-cab-compressed", + "application/x-ms-shortcut", + "application/vnd.microsoft.portable-executable", + "application/x-ms-installer", + "application/x-ms-application", + "application/x-ms-wim", + "text/x-shellscript", } def is_supported_type(head) -> bool: """Check whether the file mimetype is supported.""" mime_type = magic.Magic(mime=True).from_buffer(head) - return mime_type.startswith("image/") or mime_type in ALLOWED_MIME_TYPES - + return mime_type.startswith("image/") or mime_type not in FORBIDDEN_MIME_TYPES From 189c952ddfda2a9c124e3492a4c5019b0de8ba49 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 10 Feb 2025 10:22:51 +0100 Subject: [PATCH 3/7] reformat --- server/mergin/sync/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index ce7b0592..3d893bc1 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -366,8 +366,8 @@ def is_valid_path(filepath: str) -> bool: not len(re.split(r"\.[/\\]", filepath)) > 1 # ./ or .\ and is_valid_filepath(filepath) # invalid characters in filepath, absolute path and is_valid_filename( - os.path.basename(filepath) - ) # invalid characters in filename, reserved filenames + os.path.basename(filepath) + ) # invalid characters in filename, reserved filenames ) From 67ac1ee079baf7e9decdced537c074995a933858 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 10 Feb 2025 13:18:56 +0100 Subject: [PATCH 4/7] Move filetype check --- server/mergin/sync/public_api_controller.py | 19 ++++++------------- server/mergin/sync/utils.py | 11 ++++++++--- .../mergin/tests/test_project_controller.py | 14 ++++++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 4bea06a6..68c701ae 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -6,7 +6,6 @@ import functools import io import json -import mimetypes import os import logging from dataclasses import asdict @@ -14,6 +13,7 @@ from urllib.parse import quote import uuid from datetime import datetime + import psycopg2 from blinker import signal from connexion import NoContent, request @@ -90,6 +90,7 @@ is_valid_path, is_supported_type, is_supported_extension, + get_mimetype, ) from .errors import StorageLimitHit from ..utils import format_time_delta @@ -409,7 +410,7 @@ def download_project_file( if not is_binary(abs_path): mime_type = "text/plain" else: - mime_type = mimetypes.guess_type(abs_path)[0] + mime_type = get_mimetype(abs_path) resp.headers["Content-Type"] = mime_type resp.headers["Content-Disposition"] = "attachment; filename={}".format( quote(os.path.basename(file).encode("utf-8")) @@ -1047,6 +1048,9 @@ def push_finish(transaction_id): ) corrupted_files.append(f.path) continue + if not is_supported_type(dest_file): + logging.info(f"Rejecting blacklisted file: {dest_file}") + abort(400, f"Unsupported file type of '{f.path}' file.") if expected_size != os.path.getsize(dest_file): logging.error( @@ -1477,14 +1481,3 @@ def get_project_version(project_id: str, version: str): ).first_or_404() data = ProjectVersionSchema(exclude=["files"]).dump(pv) return data, 200 - - -def validate_file(stream, filename): - """Check file type (from its content) for unsupported types""" - buffer_for_mime_check = stream.read(2048) - if not is_supported_type(buffer_for_mime_check): - abort(400, f"Unsupported file type of '{filename}' file.") - # reset the stream position for normal reading - validated_stream = io.BytesIO(buffer_for_mime_check + stream.read()) - return validated_stream - diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index 3d893bc1..38e2fd68 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -374,7 +374,7 @@ def is_valid_path(filepath: str) -> bool: def is_supported_extension(filepath) -> bool: """Check whether file's extension is supported.""" ext = os.path.splitext(filepath)[1].lower() - return ext not in FORBIDDEN_EXTENSIONS + return ext and ext not in FORBIDDEN_EXTENSIONS FORBIDDEN_EXTENSIONS = { @@ -537,7 +537,12 @@ def is_supported_extension(filepath) -> bool: } -def is_supported_type(head) -> bool: +def is_supported_type(filepath) -> bool: """Check whether the file mimetype is supported.""" - mime_type = magic.Magic(mime=True).from_buffer(head) + mime_type = get_mimetype(filepath) return mime_type.startswith("image/") or mime_type not in FORBIDDEN_MIME_TYPES + + +def get_mimetype(filepath: str) -> str: + """Identifies file types by checking their headers""" + return magic.from_file(filepath, mime=True) diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index f26946c9..9b7633ac 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -801,7 +801,7 @@ def test_large_project_download_fail(client, diff_project): (test_project, "test.txt", "text/plain", 200), (test_project, "logo.pdf", "application/pdf", 200), (test_project, "logo.jpeg", "image/jpeg", 200), - (test_project, "base.gpkg", "application/geopackage+sqlite3", 200), + (test_project, "base.gpkg", "application/vnd.sqlite3", 200), (test_project, "json.json", "text/plain", 200), (test_project, "foo.txt", None, 404), ("bar", "test.txt", None, 404), @@ -2593,7 +2593,7 @@ def test_supported_file_upload(client): assert resp.status_code == 200 upload = Upload.query.get(resp.json["transaction"]) assert upload - # Unsupported file type is revealed in chunk_upload based on the mime type and upload is refused + # Even chunks are correctly uploaded for file in changes["added"]: for chunk_id in file["chunks"]: url = "/v1/project/push/chunk/{}/{}".format(upload.id, chunk_id) @@ -2604,7 +2604,9 @@ def test_supported_file_upload(client): resp = client.post( url, data=data, headers={"Content-Type": "application/octet-stream"} ) - assert resp.status_code == 400 - assert ( - resp.json["detail"] == f"Unsupported file type of '{spoof_name}' file." - ) + assert resp.status_code == 200 + assert resp.json["checksum"] == checksum.hexdigest() + # Unsupported file type is revealed when reconstructed from chunks - based on the mime type - and upload is refused + resp = client.post(f"/v1/project/push/finish/{upload.id}") + assert resp.status_code == 400 + assert resp.json["detail"] == f"Unsupported file type of '{spoof_name}' file." From 05189b111499e4d91101ad0655ad281405c7f0ab Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 10 Feb 2025 13:21:30 +0100 Subject: [PATCH 5/7] residue --- server/mergin/sync/public_api_controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 68c701ae..72e878d1 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -4,7 +4,6 @@ import binascii import functools -import io import json import os import logging From 4c3c647b3de0d97ea89cbb57f2d354c967cd8130 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 10 Feb 2025 13:35:45 +0100 Subject: [PATCH 6/7] Add magic lib --- server/Pipfile | 1 + server/Pipfile.lock | 132 +++++++++++++++++++++++--------------------- 2 files changed, 69 insertions(+), 64 deletions(-) diff --git a/server/Pipfile b/server/Pipfile index 62622bfc..448405ca 100644 --- a/server/Pipfile +++ b/server/Pipfile @@ -40,6 +40,7 @@ shapely = "==2.0.6" psycogreen = "==1.0.2" importlib-metadata = "==8.4.0" # https://github.com/pallets/flask/issues/4502 typing_extensions = "==4.12.2" +python-magic = "==0.4.27" # requirements for development on windows colorama = "==0.4.5" diff --git a/server/Pipfile.lock b/server/Pipfile.lock index be32d083..ee769e4c 100644 --- a/server/Pipfile.lock +++ b/server/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ade41edf9691b6241ca491897a6a989d61f5ed253b800ba068e921c5c574435a" + "sha256": "1a3dbebb1c3c4acae26310595c759edaaa9d86f2ab2984d3268ffc73a01def72" }, "pipfile-spec": 6, "requires": { @@ -917,6 +917,15 @@ "markers": "python_version >= '3.5'", "version": "==0.20.0" }, + "python-magic": { + "hashes": [ + "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", + "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.27" + }, "pytz": { "hashes": [ "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197", @@ -1590,71 +1599,66 @@ "toml" ], "hashes": [ - "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9", - "sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f", - "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273", - "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994", - "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e", - "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50", - "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e", - "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e", - "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c", - "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853", - "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8", - "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8", - "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe", - "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165", - "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb", - "sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59", - "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609", - "sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18", - "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098", - "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd", - "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3", - "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43", - "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d", - "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359", - "sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90", - "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78", - "sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a", - "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99", - "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988", - "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2", - "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0", - "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694", - "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377", - "sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d", - "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23", - "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312", - "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf", - "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6", - "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b", - "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c", - "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690", - "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a", - "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f", - "sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4", - "sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25", - "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd", - "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852", - "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0", - "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244", - "sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315", - "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078", - "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0", - "sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27", - "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132", - "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5", - "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247", - "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022", - "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b", - "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3", - "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18", - "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5", - "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f" + "sha256:050172741de03525290e67f0161ae5f7f387c88fca50d47fceb4724ceaa591d2", + "sha256:08e5fb93576a6b054d3d326242af5ef93daaac9bb52bc25f12ccbc3fa94227cd", + "sha256:09d03f48d9025b8a6a116cddcb6c7b8ce80e4fb4c31dd2e124a7c377036ad58e", + "sha256:0d03c9452d9d1ccfe5d3a5df0427705022a49b356ac212d529762eaea5ef97b4", + "sha256:13100f98497086b359bf56fc035a762c674de8ef526daa389ac8932cb9bff1e0", + "sha256:25575cd5a7d2acc46b42711e8aff826027c0e4f80fb38028a74f31ac22aae69d", + "sha256:27700d859be68e4fb2e7bf774cf49933dcac6f81a9bc4c13bd41735b8d26a53b", + "sha256:2c81e53782043b323bd34c7de711ed9b4673414eb517eaf35af92185b873839c", + "sha256:397489c611b76302dfa1d9ea079e138dddc4af80fc6819d5f5119ec8ca6c0e47", + "sha256:476f29a258b9cd153f2be5bf5f119d670d2806363595263917bddc167d6e5cce", + "sha256:4bda710139ea646890d1c000feb533caff86904a0e0638f85e967c28cb8eec50", + "sha256:4cf96beb05d004e4c51cd846fcdf9eee9eb2681518524b66b2e7610507944c2f", + "sha256:4f21e3617f48d683f30cf2a6c8b739c838e600cb1454fe6b2eb486ac2bce8fbd", + "sha256:5128f3ba694c0a1bde55fc480090392c336236c3e1a10dad40dc1ab17c7675ff", + "sha256:532fe139691af134aa8b54ed60dd3c806aa81312d93693bd2883c7b61592c840", + "sha256:5a3f7cbbcb4ad95067a6525f83a6fc78d9cbc1e70f8abaeeaeaa72ef34f48fc3", + "sha256:5b48db06f53d1864fea6dbd855e6d51d41c0f06c212c3004511c0bdc6847b297", + "sha256:5e7ac966ab110bd94ee844f2643f196d78fde1cd2450399116d3efdd706e19f5", + "sha256:5edc16712187139ab635a2e644cc41fc239bc6d245b16124045743130455c652", + "sha256:60d4ad09dfc8c36c4910685faafcb8044c84e4dae302e86c585b3e2e7778726c", + "sha256:61c834cbb80946d6ebfddd9b393a4c46bec92fcc0fa069321fcb8049117f76ea", + "sha256:6ba27a0375c5ef4d2a7712f829265102decd5ff78b96d342ac2fa555742c4f4f", + "sha256:6c96a142057d83ee993eaf71629ca3fb952cda8afa9a70af4132950c2bd3deb9", + "sha256:6d60577673ba48d8ae8e362e61fd4ad1a640293ffe8991d11c86f195479100b7", + "sha256:7eb0504bb307401fd08bc5163a351df301438b3beb88a4fa044681295bbefc67", + "sha256:8e433b6e3a834a43dae2889adc125f3fa4c66668df420d8e49bc4ee817dd7a70", + "sha256:8fa4fffd90ee92f62ff7404b4801b59e8ea8502e19c9bf2d3241ce745b52926c", + "sha256:90de4e9ca4489e823138bd13098af9ac8028cc029f33f60098b5c08c675c7bda", + "sha256:a165b09e7d5f685bf659063334a9a7b1a2d57b531753d3e04bd442b3cfe5845b", + "sha256:a46d56e99a31d858d6912d31ffa4ede6a325c86af13139539beefca10a1234ce", + "sha256:ac476e6d0128fb7919b3fae726de72b28b5c9644cb4b579e4a523d693187c551", + "sha256:ac5d92e2cc121a13270697e4cb37e1eb4511ac01d23fe1b6c097facc3b46489e", + "sha256:adc2d941c0381edfcf3897f94b9f41b1e504902fab78a04b1677f2f72afead4b", + "sha256:b6ff5be3b1853e0862da9d349fe87f869f68e63a25f7c37ce1130b321140f963", + "sha256:bb35ae9f134fbd9cf7302a9654d5a1e597c974202678082dcc569eb39a8cde03", + "sha256:be05bde21d5e6eefbc3a6de6b9bee2b47894b8945342e8663192809c4d1f08ce", + "sha256:c27df03730059118b8a923cfc8b84b7e9976742560af528242f201880879c1da", + "sha256:c7719a5e1dc93883a6b319bc0374ecd46fb6091ed659f3fbe281ab991634b9b0", + "sha256:c86f4c7a6d1a54a24d804d9684d96e36a62d3ef7c0d7745ae2ea39e3e0293251", + "sha256:ca95d40900cf614e07f00cee8c2fad0371df03ca4d7a80161d84be2ec132b7a4", + "sha256:cd4839813b09ab1dd1be1bbc74f9a7787615f931f83952b6a9af1b2d3f708bf7", + "sha256:db4b1a69976b1b02acda15937538a1d3fe10b185f9d99920b17a740a0a102e06", + "sha256:dbb1a822fd858d9853333a7c95d4e70dde9a79e65893138ce32c2ec6457d7a36", + "sha256:de6b079b39246a7da9a40cfa62d5766bd52b4b7a88cf5a82ec4c45bf6e152306", + "sha256:df6ff122a0a10a30121d9f0cb3fbd03a6fe05861e4ec47adb9f25e9245aabc19", + "sha256:e0b0f272901a5172090c0802053fbc503cdc3fa2612720d2669a98a7384a7bec", + "sha256:e2778be4f574b39ec9dcd9e5e13644f770351ee0990a0ecd27e364aba95af89b", + "sha256:e3b746fa0ffc5b6b8856529de487da8b9aeb4fb394bb58de6502ef45f3434f12", + "sha256:e642e6a46a04e992ebfdabed79e46f478ec60e2c528e1e1a074d63800eda4286", + "sha256:eafea49da254a8289bed3fab960f808b322eda5577cb17a3733014928bbfbebd", + "sha256:f0f334ae844675420164175bf32b04e18a81fe57ad8eb7e0cfd4689d681ffed7", + "sha256:f382004fa4c93c01016d9226b9d696a08c53f6818b7ad59b4e96cb67e863353a", + "sha256:f4679fcc9eb9004fdd1b00231ef1ec7167168071bebc4d66327e28c1979b4449", + "sha256:fd2fffc8ce8692ce540103dff26279d2af22d424516ddebe2d7e4d6dbb3816b2", + "sha256:ff136607689c1c87f43d24203b6d2055b42030f352d5176f9c8b204d4235ef27", + "sha256:ff52b4e2ac0080c96e506819586c4b16cdbf46724bda90d308a7330a73cc8521", + "sha256:ff562952f15eff27247a4c4b03e45ce8a82e3fb197de6a7c54080f9d4ba07845" ], "markers": "python_version >= '3.9'", - "version": "==7.6.10" + "version": "==7.6.11" }, "dill": { "hashes": [ From dfb99bc0744f7ded50786fc1ba2c53b78078fad2 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 14 Feb 2025 00:01:30 +0100 Subject: [PATCH 7/7] Update abort messages to be more helpful --- server/mergin/sync/public_api_controller.py | 13 ++++++++++--- server/mergin/tests/test_project_controller.py | 10 +++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 72e878d1..9f661a92 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -816,9 +816,16 @@ def project_push(namespace, project_name): if not all(ele.path != item.path for ele in project.files): abort(400, f"File {item.path} has been already uploaded") if not is_valid_path(item.path): - abort(400, f"File '{item.path}' contains invalid characters.") + abort( + 400, + f"Unsupported file name detected: {item.path}. Please remove the invalid characters.", + ) if not is_supported_extension(item.path): - abort(400, f"Unsupported extension of '{item.path}' file.") + abort( + 400, + f"Unsupported file type detected: {item.path}. " + f"Please remove the file or try compressing it into a ZIP file before uploading", + ) # changes' files must be unique changes_files = [ @@ -1049,7 +1056,7 @@ def push_finish(transaction_id): continue if not is_supported_type(dest_file): logging.info(f"Rejecting blacklisted file: {dest_file}") - abort(400, f"Unsupported file type of '{f.path}' file.") + abort(400, f"Unsupported file type detected: {f.path}") if expected_size != os.path.getsize(dest_file): logging.error( diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 9b7633ac..ac839d95 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -2542,7 +2542,8 @@ def test_filepath_manipulation(client): ) assert resp.status_code == 400 assert ( - resp.json["detail"] == f"File '{manipulated_path}' contains invalid characters." + resp.json["detail"] + == f"Unsupported file name detected: {manipulated_path}. Please remove the invalid characters." ) @@ -2573,7 +2574,10 @@ def test_supported_file_upload(client): headers=json_headers, ) assert resp.status_code == 400 - assert resp.json["detail"] == f"Unsupported extension of '{script_filename}' file." + assert ( + resp.json["detail"] + == f"Unsupported file type detected: {script_filename}. Please remove the file or try compressing it into a ZIP file before uploading" + ) # Extension spoofing to trick the validator spoof_name = "script.gpkg" os.rename(os.path.join(TMP_DIR, script_filename), os.path.join(TMP_DIR, spoof_name)) @@ -2609,4 +2613,4 @@ def test_supported_file_upload(client): # Unsupported file type is revealed when reconstructed from chunks - based on the mime type - and upload is refused resp = client.post(f"/v1/project/push/finish/{upload.id}") assert resp.status_code == 400 - assert resp.json["detail"] == f"Unsupported file type of '{spoof_name}' file." + assert resp.json["detail"] == f"Unsupported file type detected: {spoof_name}"