From 29a76b4c6bacae458f8fbb707e0ed49c63f8d40e Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 2 Sep 2024 20:00:51 -0700 Subject: [PATCH 1/2] Remove embedded `bbpb` code and use imported version instead --- requirements.txt | 3 +- unfurl/lib/__init__.py | 0 unfurl/lib/blackboxprotobuf/LICENSE | 21 - unfurl/lib/blackboxprotobuf/__init__.py | 4 - unfurl/lib/blackboxprotobuf/lib/__init__.py | 4 - unfurl/lib/blackboxprotobuf/lib/api.py | 452 -------------- unfurl/lib/blackboxprotobuf/lib/config.py | 26 - unfurl/lib/blackboxprotobuf/lib/exceptions.py | 51 -- unfurl/lib/blackboxprotobuf/lib/protofile.py | 450 -------------- .../blackboxprotobuf/lib/types/__init__.py | 4 - .../lib/blackboxprotobuf/lib/types/fixed.py | 111 ---- .../lib/types/length_delim.py | 587 ------------------ .../blackboxprotobuf/lib/types/type_maps.py | 89 --- .../lib/blackboxprotobuf/lib/types/varint.py | 98 --- unfurl/parsers/parse_protobuf.py | 2 +- 15 files changed, 3 insertions(+), 1899 deletions(-) delete mode 100644 unfurl/lib/__init__.py delete mode 100644 unfurl/lib/blackboxprotobuf/LICENSE delete mode 100644 unfurl/lib/blackboxprotobuf/__init__.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/__init__.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/api.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/config.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/exceptions.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/protofile.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/types/__init__.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/types/fixed.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/types/length_delim.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/types/type_maps.py delete mode 100644 unfurl/lib/blackboxprotobuf/lib/types/varint.py diff --git a/requirements.txt b/requirements.txt index 9240ac9..38ad92f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ pycountry pymispwarninglists>=1.5 requests torf -ulid-py \ No newline at end of file +ulid-py +bbpb \ No newline at end of file diff --git a/unfurl/lib/__init__.py b/unfurl/lib/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/unfurl/lib/blackboxprotobuf/LICENSE b/unfurl/lib/blackboxprotobuf/LICENSE deleted file mode 100644 index a7abbb9..0000000 --- a/unfurl/lib/blackboxprotobuf/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018 NCC Group Plc - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/unfurl/lib/blackboxprotobuf/__init__.py b/unfurl/lib/blackboxprotobuf/__init__.py deleted file mode 100644 index 30a923f..0000000 --- a/unfurl/lib/blackboxprotobuf/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""This module provides generic functions for encoding and decoding protobuf's - to python dictionaries or JSON -""" -from unfurl.lib.blackboxprotobuf.lib.api import * diff --git a/unfurl/lib/blackboxprotobuf/lib/__init__.py b/unfurl/lib/blackboxprotobuf/lib/__init__.py deleted file mode 100644 index d42f5c7..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""This module provides generic functions for encoding and decoding protobufs - to python dictionaries or JSON -""" -from .api import * diff --git a/unfurl/lib/blackboxprotobuf/lib/api.py b/unfurl/lib/blackboxprotobuf/lib/api.py deleted file mode 100644 index 21031be..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/api.py +++ /dev/null @@ -1,452 +0,0 @@ -"""Methods for easy encoding and decoding of messages""" - -import re -import six -import json -import collections -import unfurl.lib.blackboxprotobuf.lib.types.length_delim -import unfurl.lib.blackboxprotobuf.lib.config -from unfurl.lib.blackboxprotobuf.lib.exceptions import TypedefException - - -def decode_message(buf, message_type=None, config=None): - """Decode a message to a Python dictionary. - Returns tuple of (values, types) - """ - - if config is None: - config = unfurl.lib.blackboxprotobuf.lib.config.default - - if isinstance(buf, bytearray): - buf = bytes(buf) - buf = six.ensure_binary(buf) - if message_type is None or isinstance(message_type, str): - if message_type not in config.known_types: - message_type = {} - else: - message_type = config.known_types[message_type] - - value, typedef, _ = unfurl.lib.blackboxprotobuf.lib.types.length_delim.decode_message( - buf, config, message_type - ) - return value, typedef - - -# TODO add explicit validation of values to message type -def encode_message(value, message_type, config=None): - """Encodes a python dictionary to a message. - Returns a bytearray - """ - - if config is None: - config = unfurl.lib.blackboxprotobuf.lib.config.default - return bytes( - unfurl.lib.blackboxprotobuf.lib.types.length_delim.encode_message( - value, config, message_type - ) - ) - - -def protobuf_to_json(*args, **kwargs): - """Encode to python dictionary and dump to JSON. - Takes same arguments as decode_message - """ - value, message_type = decode_message(*args, **kwargs) - value = json_safe_transform( - value, message_type, False, config=kwargs.get("config", None) - ) - value = sort_output(value, message_type, config=kwargs.get("config", None)) - _annotate_typedef(message_type, value) - message_type = sort_typedef(message_type) - return json.dumps(value, indent=2), message_type - - -def protobuf_from_json(json_str, message_type, *args, **kwargs): - """Decode JSON string to JSON and then to protobuf. - Takes same arguments as encode_message - """ - value = json.loads(json_str) - _strip_typedef_annotations(message_type) - value = json_safe_transform(value, message_type, True) - return encode_message(value, message_type, *args, **kwargs) - - -def export_protofile(message_types, output_filename): - """Export the give messages as ".proto" file. - Expects a dictionary with {message_name: typedef} and a filename to - write to. - """ - unfurl.lib.blackboxprotobuf.lib.protofile.export_proto( - message_types, output_filename=output_filename - ) - - -def import_protofile(input_filename, save_to_known=True, config=None): - """import ".proto" files and returns the typedef map - Expects a filename for the ".proto" file - """ - if config is None: - config = unfurl.lib.blackboxprotobuf.lib.config.default - - new_typedefs = unfurl.lib.blackboxprotobuf.lib.protofile.import_proto( - config, input_filename=input_filename - ) - if save_to_known: - config.known_types.update(new_typedefs) - else: - return new_typedefs - - -NAME_REGEX = re.compile(r"\A[a-zA-Z][a-zA-Z0-9_]*\Z") - - -def validate_typedef(typedef, old_typedef=None, path=None, config=None): - """Validate the typedef format. Optionally validate wiretype of a field - number has not been changed - """ - if path is None: - path = [] - if config is None: - config = unfurl.lib.blackboxprotobuf.lib.config.default - - int_keys = set() - field_names = set() - for field_number, field_typedef in typedef.items(): - alt_field_number = None - if "-" in str(field_number): - field_number, alt_field_number = field_number.split("-") - - # Validate field_number is a number - if not str(field_number).isdigit(): - raise TypedefException("Field number must be a digit: %s" % field_number) - field_number = str(field_number) - - field_path = path[:] - field_path.append(field_number) - - # Check for duplicate field numbers - if field_number in int_keys: - raise TypedefException( - "Duplicate field number: %s" % field_number, field_path - ) - int_keys.add(field_number) - - # Must have a type field - if "type" not in field_typedef: - raise TypedefException( - "Field number must have a type value: %s" % field_number, field_path - ) - if alt_field_number is not None: - if field_typedef["type"] != "message": - raise TypedefException( - "Alt field number (%s) specified for non-message field: %s" - % (alt_field_number, field_number), - field_path, - ) - - valid_type_fields = [ - "type", - "name", - "message_typedef", - "message_type_name", - "alt_typedefs", - "example_value_ignored", - "seen_repeated", - ] - for key, value in field_typedef.items(): - # Check field keys against valid values - if key not in valid_type_fields: - raise TypedefException( - 'Invalid field key "%s" for field number %s' % (key, field_number), - field_path, - ) - if ( - key in ["message_typedef", "message_type_name"] - and not field_typedef["type"] == "message" - ): - raise TypedefException( - 'Invalid field key "%s" for field number %s' % (key, field_number), - field_path, - ) - if key == "group_typedef" and not field_typedef["type"] == "group": - raise TypedefException( - 'Invalid field key "%s" for field number %s' % (key, field_number), - field_path, - ) - - # Validate type value - if key == "type": - if value not in unfurl.lib.blackboxprotobuf.lib.types.type_maps.WIRETYPES: - raise TypedefException( - 'Invalid type "%s" for field number %s' % (value, field_number), - field_path, - ) - # Check for duplicate names - if key == "name": - if value.lower() in field_names: - raise TypedefException( - ('Duplicate field name "%s" for field ' "number %s") - % (value, field_number), - field_path, - ) - if not NAME_REGEX.match(value): - raise TypedefException( - ( - 'Invalid field name "%s" for field ' - "number %s. Should match %s" - ) - % (value, field_number, "[a-zA-Z_][a-zA-Z0-9_]*"), - field_path, - ) - if value != "": - field_names.add(value.lower()) - - # Check if message type name is known - if key == "message_type_name": - if value not in config.known_types: - raise TypedefException( - ( - 'Message type "%s" for field number' - " %s is not known. Known types: %s" - ) - % (value, field_number, config.known_types.keys()), - field_path, - ) - - # Recursively validate inner typedefs - if key in ["message_typedef", "group_typedef"]: - if old_typedef is None: - validate_typedef(value, path=field_path, config=config) - else: - validate_typedef( - value, old_typedef[field_number][key], path=field_path - ) - if key == "alt_typedefs": - for alt_field_number, alt_typedef in value.items(): - # TODO validate alt_typedefs against old typedefs? - validate_typedef(alt_typedef, path=field_path, config=config) - - if old_typedef is not None: - wiretype_map = {} - for field_number, value in old_typedef.items(): - wiretype_map[ - int(field_number) - ] = unfurl.lib.blackboxprotobuf.lib.types.type_maps.WIRETYPES[value["type"]] - for field_number, value in typedef.items(): - field_path = path[:] - field_path.append(str(field_number)) - if int(field_number) in wiretype_map: - old_wiretype = wiretype_map[int(field_number)] - if ( - old_wiretype - != unfurl.lib.blackboxprotobuf.lib.types.type_maps.WIRETYPES[value["type"]] - ): - raise TypedefException( - ( - "Wiretype for field number %s does" - " not match old type definition" - ) - % field_number, - field_path, - ) - - -def json_safe_transform(values, typedef, toBytes, config=None): - """JSON doesn't handle bytes type well. We want to go through and encode - every bytes type as latin1 to get a semi readable text""" - - if config is None: - config = unfurl.lib.blackboxprotobuf.lib.config.default - name_map = { - item["name"]: number - for number, item in typedef.items() - if item.get("name", None) - } - for name, value in values.items(): - alt_number = None - base_name = name - if "-" in name: - base_name, alt_number = name.split("-") - - if base_name in name_map: - field_number = name_map[base_name] - else: - field_number = base_name - - field_type = typedef[field_number]["type"] - if field_type == "message": - field_typedef = _get_typedef_for_message(typedef[field_number], config) - if alt_number is not None: - # if we have an alt type, then let's look that up instead - if alt_number not in typedef[field_number].get("alt_typedefs", {}): - raise TypedefException( - ( - "Provided alt field name/number " - "%s is not valid for field_number %s" - ) - % (alt_number, field_number) - ) - field_type = typedef[field_number]["alt_typedefs"][alt_number] - if isinstance(field_type, dict): - field_typedef = field_type - field_type = "message" - - is_list = isinstance(value, list) - field_values = value if is_list else [value] - for i, field_value in enumerate(field_values): - if field_type == "bytes": - if toBytes: - field_values[i] = field_value.encode("latin1") - else: - field_values[i] = field_value.decode("latin1") - elif field_type == "message": - field_values[i] = json_safe_transform( - field_value, - field_typedef, - toBytes, - config=config, - ) - - # convert back to single value if needed - if not is_list: - values[name] = field_values[0] - else: - values[name] = field_values - return values - - -def _get_typedef_for_message(field_typedef, config): - assert field_typedef["type"] == "message" - if "message_typedef" in field_typedef: - return field_typedef["message_typedef"] - elif field_typedef.get("message_type_name"): - if field_typedef["message_type_name"] not in config.known_types: - raise TypedefException( - "Got 'message_type_name' not in known_messages: %s" - % field_typedef["message_type_name"] - ) - return config.known_types[field_typedef["message_type_name"]] - else: - raise TypedefException( - "Got 'message' type without typedef or type name: %s" % field_typedef - ) - - -def sort_output(value, typedef, config=None): - """Sort output by the field number in the typedef. Helps with readability - in a JSON dump""" - output_dict = collections.OrderedDict() - if config is None: - config = unfurl.lib.blackboxprotobuf.lib.config.default - - # Make a list of all the field names we have, aggregate together the alt fields as well - field_names = {} - for field_name in value.keys(): - if "-" in field_name: - field_name_base, alt_number = field_name.split("-") - else: - field_name_base = field_name - alt_number = None - field_names.setdefault(field_name_base, []).append((field_name, alt_number)) - - for field_number, field_def in sorted(typedef.items(), key=lambda t: int(t[0])): - field_number = str(field_number) - seen_field_names = field_names.get(field_number, []) - - # Try getting matching fields by name as well - if field_def.get("name", "") != "": - field_name = field_def["name"] - seen_field_names.extend(field_names.get(field_name, [])) - - for field_name, alt_number in seen_field_names: - field_type = field_def["type"] - field_message_typedef = None - if field_type == "message": - field_message_typedef = _get_typedef_for_message(field_def, config) - - if alt_number is not None: - if alt_number not in field_def["alt_typedefs"]: - raise TypedefException( - ( - "Provided alt field name/number " - "%s is not valid for field_number %s" - ) - % (alt_number, field_number) - ) - field_type = field_def["alt_typedefs"][alt_number] - if isinstance(field_type, dict): - field_message_typedef = field_type - field_type = "message" - - if field_type == "message": - if not isinstance(value[field_name], list): - output_dict[field_name] = sort_output( - value[field_name], field_message_typedef - ) - else: - output_dict[field_name] = [] - for field_value in value[field_name]: - output_dict[field_name].append( - sort_output(field_value, field_message_typedef) - ) - else: - output_dict[field_name] = value[field_name] - - return output_dict - - -def sort_typedef(typedef): - """Sort output by field number and sub_keys so name then type is first""" - - TYPEDEF_KEY_ORDER = ["name", "type", "message_type_name", "example_value_ignored"] - output_dict = collections.OrderedDict() - - for field_number, field_def in sorted(typedef.items(), key=lambda t: int(t[0])): - output_field_def = collections.OrderedDict() - field_def = field_def.copy() - for key in TYPEDEF_KEY_ORDER: - if key in field_def: - output_field_def[key] = field_def[key] - del field_def[key] - for key, value in field_def.items(): - - # TODO handle alt typedefs - if key == "message_typedef": - output_field_def[key] = sort_typedef(value) - else: - output_field_def[key] = value - output_dict[field_number] = output_field_def - return output_dict - - -def _annotate_typedef(typedef, message): - """Add values from message into the typedef so it's easier to figure out - which field when you're editing manually""" - - for field_number, field_def in typedef.items(): - field_value = None - field_name = str(field_number) - if field_name not in message and field_def.get("name", "") != "": - field_name = field_def["name"] - - if field_name in message: - field_value = message[field_name] - - # TODO handle alt typedefs - if field_def["type"] == "message": - if isinstance(field_value, list): - for value in field_value: - _annotate_typedef(field_def["message_typedef"], value) - else: - _annotate_typedef(field_def["message_typedef"], field_value) - else: - field_def["example_value_ignored"] = field_value - - -def _strip_typedef_annotations(typedef): - """Remove example values placed by _annotate_typedef""" - for _, field_def in typedef.items(): - if "example_value_ignored" in field_def: - del field_def["example_value_ignored"] - if "message_typedef" in field_def: - _strip_typedef_annotations(field_def["message_typedef"]) diff --git a/unfurl/lib/blackboxprotobuf/lib/config.py b/unfurl/lib/blackboxprotobuf/lib/config.py deleted file mode 100644 index a6a56eb..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/config.py +++ /dev/null @@ -1,26 +0,0 @@ -from .types import type_maps - - -class Config: - def __init__(self): - - # Map of message type names to typedefs, previously stored at - # `blackboxprotobuf.known_messages` - self.known_types = {} - - # Default type for "bytes" like objects that aren't messages or strings - # Other option is currently just 'bytes_hex' - self.default_binary_type = "bytes" - - # Change the default type for a wiretype (eg. - self.default_types = {} - - def get_default_type(self, wiretype): - default_type = self.default_types.get(wiretype, None) - if default_type is None: - default_type = type_maps.WIRE_TYPE_DEFAULTS.get(wiretype, None) - - return default_type - - -default = Config() diff --git a/unfurl/lib/blackboxprotobuf/lib/exceptions.py b/unfurl/lib/blackboxprotobuf/lib/exceptions.py deleted file mode 100644 index 0e13728..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/exceptions.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Exception classes for BlackboxProtobuf""" - - -class BlackboxProtobufException(Exception): - """Base class for excepions raised by Blackbox Protobuf""" - - def __init__(self, message, path=None, *args): - self.path = path - super(BlackboxProtobufException, self).__init__(message, *args) - - def set_path(self, path): - if self.path is None: - self.path = path - - -class TypedefException(BlackboxProtobufException): - """Thrown when an error is identified in the type definition, such as - conflicting or incosistent values.""" - - def __str__(self): - message = super(TypedefException, self).__str__() - if self.path is not None: - message = ( - "Encountered error within typdef for field %s: " - % "->".join(map(str, self.path)) - ) + message - return message - - -class EncoderException(BlackboxProtobufException, ValueError): - """Thrown when there is an error encoding a dictionary to a type definition""" - - def __str__(self): - message = super(EncoderException, self).__str__() - if self.path is not None: - message = ( - "Encountered error encoding field %s: " % "->".join(map(str, self.path)) - ) + message - return message - - -class DecoderException(BlackboxProtobufException, ValueError): - """Thrown when there is an error decoding a bytestring to a dictionary""" - - def __str__(self): - message = super(DecoderException, self).__str__() - if self.path is not None: - message = ( - "Encountered error decoding field %s: " % "->".join(map(str, self.path)) - ) + message - return message diff --git a/unfurl/lib/blackboxprotobuf/lib/protofile.py b/unfurl/lib/blackboxprotobuf/lib/protofile.py deleted file mode 100644 index c910357..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/protofile.py +++ /dev/null @@ -1,450 +0,0 @@ -""" -Python methods for importing and exporting '.proto' files from the BBP type -definition format. -""" - -# TODO get custom exceptions for these methods - -import io -import re -import logging -from unfurl.lib.blackboxprotobuf.lib.exceptions import TypedefException -from unfurl.lib.blackboxprotobuf.lib.api import sort_typedef - -PROTO_FILE_TYPE_MAP = { - "uint": "uint64", - "int": "int64", - "sint": "sint64", - "fixed32": "fixed32", - "sfixed32": "sfixed32", - "float": "float", - "fixed64": "fixed64", - "sfixed64": "sfixed64", - "double": "double", - "bytes": "bytes", - "bytes_hex": "bytes", - "string": "string", -} - -PACKABLE_TYPES = [ - "uint", - "int", - "sint", - "fixed32", - "sfixed32", - "float", - "fixed64", - "sfixed64", - "double", -] - -# Inverse of the above, but we have to include more types -PROTO_FILE_TYPE_TO_BBP = { - "double": "double", - "float": "float", - "int32": "int", - "int64": "int", - "uint32": "uint", - "uint64": "uint", - "sint32": "sint", - "sint64": "sint", - "fixed32": "fixed32", - "fixed64": "fixed64", - "sfixed32": "sfixed32", - "sfixed64": "sfixed64", - "bool": "uint", - "string": "string", - # should be default_binary_type, but can't handle that well here - "bytes": "bytes", -} - -NAME_REGEX = re.compile(r"\A[a-zA-Z_][a-zA-Z0-9_]*\Z") - -# add packed types to the list -for packable_type in PACKABLE_TYPES: - packed_type = "packed_" + packable_type - PROTO_FILE_TYPE_MAP[packed_type] = PROTO_FILE_TYPE_MAP[packable_type] - - -def _print_message(message_name, typedef, output_file, depth=0): - indent = u" " * depth - if not NAME_REGEX.match(message_name): - raise TypedefException("Message name: %s is not valid" % message_name) - - # sort typedef for better looking output - typedef = sort_typedef(typedef) - - message_name = message_name.strip() - output_file.write(u"\n") - output_file.write(indent) - output_file.write(u"message %s {\n" % message_name) - for field_number, field_typedef in typedef.items(): - # TODO Default to all fields as repeated? or optional - proto_type = None - field_name = None - field_options = "" - - # a repeated field with one element is indistinduishable from a - # repeated field so we just put repeated if we have proof that it is - # repeatable, but this might be wrong sometimes - # maybe some sort of protobuf discovery tool can detect this - is_repeated = field_typedef.get("seen_repeated", False) - - if "name" in field_typedef and field_typedef["name"] != "": - field_name = field_typedef["name"] - field_name = field_name.strip() - if not NAME_REGEX.match(field_name): - field_name = None - if field_name is None: - field_name = u"field%s" % field_number - - if field_typedef["type"] == "message": - # If we have multiple typedefs, this means is something like the Any - # message, and has to be manually reparsed to each type - if "alt_typedefs" in field_typedef: - proto_type = "bytes" - else: - proto_type = field_name + "_type" - _print_message( - proto_type, field_typedef["message_typedef"], output_file, depth + 1 - ) - else: - if field_typedef["type"] not in PROTO_FILE_TYPE_MAP: - raise TypedefException( - "Type %s does not have a mapping to protobuf types." - % field_typedef["type"] - ) - proto_type = PROTO_FILE_TYPE_MAP[field_typedef["type"]] - - # we're using proto3 syntax. Repeated numeric fields are packed by default - # if it's repeated and not packed, then make sure we specify it's not packed - if is_repeated and field_typedef["type"] in PACKABLE_TYPES: - field_options = u" [packed=false]" - # if it's a packed type, we'll explicitoly set that too, can't hurt - elif field_typedef["type"].startswith("packed_"): - field_options = u" [packed=true]" - is_repeated = True - - output_file.write(indent) - output_file.write( - u" %s%s %s = %s%s;\n" - % ( - "repeated " if is_repeated else "", - proto_type, - field_name, - field_number, - field_options, - ) - ) - - output_file.write(indent) - output_file.write(u"}\n\n") - - -def export_proto(typedef_map, output_filename=None, output_file=None, package=None): - """Export the given type definitons as a '.proto' file. Typedefs are - expected as a dictionary of {'message_name': typedef } - - Write to output_file or output_filename if provided, otherwise return a string - output_filename will be overwritten if it exists - """ - return_string = False - if output_filename is not None: - output_file = io.open(output_filename, "w+") - - if output_file is None: - return_string = True - output_file = io.StringIO() - - # preamble - output_file.write(u'syntax = "proto3";\n\n') - if package: - output_file.write(u"package %s;\n\n" % package) - - for typedef_name, typedef in typedef_map.items(): - _print_message(typedef_name, typedef, output_file) - - if return_string: - return output_file.getvalue() - # close the file if we opened it - elif output_filename is not None: - output_file.close() - return None - - -MESSAGE_START_REGEX = re.compile(r"^message +([a-zA-Z_0-9]+) *{.*") -FIELD_REGEX = re.compile( - r"^ *(repeated|optional|required)? *([a-zA-Z0-9_]+) +([a-zA-Z0-9_]+) += +([0-9]+) *(\[[a-z]+=[a-z]*\])?.*;.*$" -) -SYNTAX_REGEX = re.compile(r'^ *syntax += +"(proto\d)" *;.*') -ENUM_REGEX = re.compile(r"^ *enum +([a-zA-Z0-9_]+) *{.*") -PACKAGE_REGEX = re.compile(r"^ *package +([a-zA-Z0-9_.]+) *;.*") - - -def import_proto(config, input_string=None, input_filename=None, input_file=None): - typedef_map = {} - if input_string is not None: - input_file = io.StringIO(input_string) - if input_file is None and input_filename is not None: - input_file = io.open(input_filename, "r") - if input_file is None: - raise ValueError("No file provided to import_proto") - - syntax_version = "proto2" - package_prefix = "" - enum_names = [] - message_trees = [] - message_names = [] - - line = input_file.readline() - while line: - line = line.strip() - - if line.startswith("syntax") and SYNTAX_REGEX.match(line): - syntax_version = SYNTAX_REGEX.match(line).group(1) - - elif line.startswith("package") and PACKAGE_REGEX.match(line): - package_prefix = PACKAGE_REGEX.match(line).group(1) + "." - - elif line.startswith("import"): - logging.warn( - "Proto file has import which is not supported " - "by the parser. Ensure the imported files are " - "processed first: %s", - line, - ) - elif line.startswith("enum") and ENUM_REGEX.match(line): - enum_name = _parse_enum(line, input_file) - enum_names.append(enum_name) - - elif line.startswith("message") and MESSAGE_START_REGEX.match(line): - message_tree = _preparse_message(line, input_file) - message_trees.append(message_tree) - - line = input_file.readline() - - # TODO parse the message data - for tree in message_trees: - new_message_names, new_enum_names = _collect_names(package_prefix, tree) - enum_names += new_enum_names - message_names += new_message_names - - logging.debug("Got the following enum_names: %s", enum_names) - logging.debug("Got the following message_names: %s", message_names) - - for tree in message_trees: - _parse_message( - tree, - typedef_map, - message_names, - enum_names, - package_prefix, - syntax_version == "proto3", - config, - ) - - return typedef_map - - -def _parse_enum(line, input_file): - """Parse an enum out of the file. Goes from enum declaration to next } - Returns the enum's name - """ - enum_name = ENUM_REGEX.match(line).group(1) - # parse until the next '}' - while "}" not in line: - line = input_file.readline() - if not line: - raise ValueError("Did not find close of enum") - return enum_name - - -def _preparse_message(line, input_file): - """Parse out a message name and the lines that make it up""" - message_name = MESSAGE_START_REGEX.match(line).group(1) - message_lines = [] - inner_enums = [] - inner_messages = [] - - while "}" not in line: - line = input_file.readline() - if not line: - raise ValueError("Did not find close of message") - - line = line.strip() - if line.startswith("enum") and ENUM_REGEX.match(line): - enum_name = _parse_enum(line, input_file) - inner_enums.append(enum_name) - - elif line.startswith("message") and MESSAGE_START_REGEX.match(line): - message_tree = _preparse_message(line, input_file) - inner_messages.append(message_tree) - # not an inner enum or message - else: - message_lines.append(line) - - return { - "name": message_name, - "data": message_lines, - "enums": inner_enums, - "inner_messages": inner_messages, - } - - -def _collect_names(prefix, message_tree): - message_names = [] - enum_names = [] - - name = prefix + message_tree["name"] - message_names.append(name) - for enum_name in message_tree["enums"]: - enum_names.append(prefix + enum_name) - for inner_message in message_tree["inner_messages"]: - new_message_names, new_enum_names = _collect_names(name + ".", inner_message) - message_names += new_message_names - enum_names += new_enum_names - return message_names, enum_names - - -def _check_message_name(current_path, name, known_message_names, config): - # Verify message name against preparsed message names and global - # known_messages - # For example, if we have: - # Message.InnerMesage - # referenced from: - # PackageA.Message2 - # we would look up: - # PackageA.Message2.Message.InnerMessage - # PackageA.Message.InnerMessage - # should also work for enums - if name in config.known_types: - return True - # search for anything under a common prefix in known_message_names - logging.debug("Testing message name: %s", name) - - prefix_options = [""] - for part in current_path.split("."): - if part: - prefix_options = [prefix_options[0] + part + "."] + prefix_options - - logging.debug("prefix_options: %s", prefix_options) - for prefix in prefix_options: - logging.debug("Testing message name: %s", prefix + name) - if prefix + name in known_message_names: - return prefix + name - # remove the last bit of the prefix - if "." not in prefix: - break - prefix = ".".join(prefix.split(".")[:-1]) - logging.debug( - "Message %s not found from %s Known names are: %s", - name, - current_path, - known_message_names, - ) - return None - - -def _parse_message( - message_tree, typdef_map, known_message_names, enum_names, prefix, is_proto3, config -): - message_typedef = {} - message_name = prefix + message_tree["name"] - prefix = message_name + "." - # parse the actual message fields - for line in message_tree["data"]: - # lines should already be stripped and should not have messages or enums - # logging.debug("Line before assert: %s", line) - assert all([not line.strip().startswith(x) for x in ["message ", "enum "]]) - # Check if the line matches the field regex - match = FIELD_REGEX.match(line) - if match: - field_number, field_typedef = _parse_field( - match, known_message_names, enum_names, prefix, is_proto3, config - ) - message_typedef[field_number] = field_typedef - - # add the messsage to tyep returned typedefs - logging.debug("Adding message %s to typedef maps", message_name) - typdef_map[message_name] = message_typedef - - for inner_message in message_tree["inner_messages"]: - # TODO prefix should be added to? - _parse_message( - inner_message, - typdef_map, - known_message_names, - enum_names, - prefix, - is_proto3, - config, - ) - - -# parse a field into a dictionary for the typedef -def _parse_field(match, known_message_names, enum_names, prefix, is_proto3, config): - typedef = {} - - field_name = match.group(3) - if not field_name: - raise ValueError("Could not parse field name from line: %s" % match) - typedef["name"] = field_name - field_number = match.group(4) - if not field_number: - raise ValueError("Could not parse field number from line: %s" % match) - - # figure out repeated - field_rule = match.group(1) - is_repeated = False - if field_rule and "repeated" in field_rule: - is_repeated = True - typedef["seen_repeated"] = True - - field_type = match.group(2) - if not field_type: - raise ValueError("Could not parse field type from line: %s" % match) - # check normal types - bbp_type = PROTO_FILE_TYPE_TO_BBP.get(field_type, None) - if not bbp_type: - logging.debug("Got non-basic type: %s, checking enums", field_type) - # check enum names - if _check_message_name(prefix, field_type, enum_names, config): - # enum = uint - bbp_type = "uint" - if not bbp_type: - # Not enum or normal type, check messages - message_name = _check_message_name( - prefix, field_type, known_message_names, config - ) - if message_name: - bbp_type = "message" - typedef["message_type_name"] = message_name - - if not bbp_type: - # If we don't have a type now, then fail - raise ValueError( - "Could not get a type for field %s: %s" % (field_name, field_type) - ) - - # figure out packed - # default based on repeated + proto3, fallback to options - field_options = match.group(5) - is_packed = is_repeated and is_proto3 and (field_type in PACKABLE_TYPES) - if is_packed and field_options and "packed=false" in field_options: - is_packed = False - elif is_repeated and field_options and "packed=true" in field_options: - is_packed = True - - # make sure the type lines up with packable - if is_packed and bbp_type not in PACKABLE_TYPES: - raise ValueError( - "Field %s set as packable, but not a packable type: %s" - % (field_name, bbp_type) - ) - if is_packed: - bbp_type = "packed_" + bbp_type - - typedef["type"] = bbp_type - - logging.debug("Parsed field number %s: %s", field_number, typedef) - return field_number, typedef diff --git a/unfurl/lib/blackboxprotobuf/lib/types/__init__.py b/unfurl/lib/blackboxprotobuf/lib/types/__init__.py deleted file mode 100644 index 5a579e8..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/types/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""This module contains classes/methods for decoding individual types. Grouped - into general categories of wire type -""" -from .type_maps import * diff --git a/unfurl/lib/blackboxprotobuf/lib/types/fixed.py b/unfurl/lib/blackboxprotobuf/lib/types/fixed.py deleted file mode 100644 index 25aa722..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/types/fixed.py +++ /dev/null @@ -1,111 +0,0 @@ -""" Functions for encoding and decoding fixed size structs """ -import struct -import binascii -import six -from unfurl.lib.blackboxprotobuf.lib.exceptions import DecoderException, EncoderException - -# Generic functions for encoding/decoding structs based on the "struct" format -def encode_struct(fmt, value): - """Generic method for encoding arbitrary python "struct" values""" - try: - return struct.pack(fmt, value) - except struct.error as exc: - six.raise_from( - EncoderException( - "Error encoding value %r with format string %s" % (value, fmt) - ), - exc, - ) - - -def decode_struct(fmt, buf, pos): - """Generic method for decoding arbitrary python "struct" values""" - new_pos = pos + struct.calcsize(fmt) - try: - return struct.unpack(fmt, buf[pos:new_pos])[0], new_pos - except struct.error as exc: - six.raise_from( - DecoderException( - "Error deocding format string %s from bytes: %s" - % (fmt, binascii.hexlify(buf[pos:new_pos])) - ), - exc, - ) - - -_fixed32_fmt = ", []) - } - """ - - output_map = {} - while pos < end: - # Read in a field - try: - if six.PY2: - tag, pos = decoder._DecodeVarint(str(buf), pos) - else: - tag, pos = decoder._DecodeVarint(buf, pos) - except (IndexError, decoder._DecodeError) as exc: - six.raise_from( - DecoderException( - "Error decoding length from buffer: %r..." - % (binascii.hexlify(buf[pos : pos + 8])), - path=path, - ), - exc, - ) - - field_number, wire_type = wire_format.UnpackTag(tag) - - # We want field numbers as strings everywhere - field_number = str(field_number) - - path = path[:] + [field_number] - - if field_number in output_map and output_map[field_number][0] != wire_type: - """This should never happen""" - raise DecoderException( - "Field %s has mistmatched wiretypes. Previous: %s Now: %s" - % (field_number, output_map[field_number][0], wire_type), - path=path, - ) - - length = None - if wire_type == wire_format.WIRETYPE_VARINT: - # We actually have to read in the whole varint to figure out it's size - _, new_pos = varint.decode_varint(buf, pos) - length = new_pos - pos - elif wire_type == wire_format.WIRETYPE_FIXED32: - length = 4 - elif wire_type == wire_format.WIRETYPE_FIXED64: - length = 8 - elif wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED: - # Read the length from the start of the message - # add on the length of the length tag as well - bytes_length, new_pos = varint.decode_varint(buf, pos) - length = bytes_length + (new_pos - pos) - elif wire_type in [ - wire_format.WIRETYPE_START_GROUP, - wire_format.WIRETYPE_END_GROUP, - ]: - raise DecoderException("GROUP wire types not supported", path=path) - else: - raise DecoderException("Got unkown wire type: %d" % wire_type, path=path) - if pos + length > end: - raise DecoderException( - "Decoded length for field %s goes over end: %d > %d" - % (field_number, pos + length, end), - path=path, - ) - - field_buf = buf[pos : pos + length] - - if field_number in output_map: - output_map[field_number][1].append(field_buf) - else: - output_map[field_number] = (wire_type, [field_buf]) - pos += length - return output_map, pos - - -def _get_field_key(field_number, typedef, path): - """If field_number has a name, then use that""" - if not isinstance(field_number, (int, str)): - # Should only get unpredictable inputs from encoding - raise EncoderException("Field key in message must be a str or int", path=path) - if isinstance(field_number, int): - field_number = str(field_number) - alt_field_number = None - if "-" in field_number: - field_number, alt_field_number = field_number.split("-") - # TODO - raise NotImplementedError( - "Handling for _get_field_key not implemented for alt typedefs: %s" - % field_number, - path=path, - ) - if field_number in typedef and "name" in typedef[field_number]: - field_key = typedef[field_number]["name"] - else: - field_key = field_number - # Return the new field_name + alt_field_number - return field_key + ("" if alt_field_number is None else "-" + alt_field_number) - - -def _try_decode_lendelim_fields( - buffers, field_key, field_typedef, message_output, config -): - # This is where things get weird - # To start, since we want to decode messages and not treat every - # embedded message as bytes, we have to guess if it's a message or - # not. - # Unlike other types, we can't assume our message types are - # consistent across the tree or even within the same message. - # A type could be a bytes type that might decode to different - # messages that don't have the same type definition. This is where - # 'alt_typedefs' let us say, these are the different message types we've - # seen for this one field. - # In general, if something decodes as a message once, the rest should too - # and we can enforce that across a single message, but not multiple - # messages. - # This is going to change the definition of "alt_typedefs" a bit from just - # alternate message type definitions to also allowing downgrading to - # 'bytes' or string with an 'alt_type' if it doesn't parse - - try: - outputs_map = {} - # grab all dictonary alt_typedefs - all_typedefs = { - # we don't want this to modify in-place if it fails - key: copy.deepcopy(value) - for key, value in field_typedef.get("alt_typedefs", {}).items() - if isinstance(value, dict) - } - all_typedefs["1"] = copy.deepcopy(field_typedef.get("message_typedef", {})) - - for buf in buffers: - output = None - output_typedef = None - output_typedef_num = None - for alt_typedef_num, alt_typedef in sorted( - all_typedefs.items(), key=lambda x: int(x[0]) - ): - try: - output, output_typedef, _ = decode_lendelim_message( - buf, config, alt_typedef - ) - except: - continue - output_typedef_num = alt_typedef_num - break - # try an anonymous type - # let the error propogate up if we fail this - if output is None: - output, output_typedef, _ = decode_lendelim_message(buf, config, {}) - output_typedef_num = str( - max([int(i) for i in ["0"] + list(all_typedefs.keys())]) + 1 - ) - - # save the output or typedef we found - all_typedefs[output_typedef_num] = output_typedef - output_list = outputs_map.get(output_typedef_num, []) - output_list.append(output) - outputs_map[output_typedef_num] = output_list - # was able to decode everything as a message - field_typedef["type"] = "message" - field_typedef["message_typedef"] = all_typedefs["1"] - if len(all_typedefs.keys()) > 1: - del all_typedefs["1"] - field_typedef.setdefault("alt_typedefs", {}).update(all_typedefs) - # messages get set as "key-alt_number" - for output_typedef_num, outputs in outputs_map.items(): - output_field_key = field_key - if output_typedef_num != "1": - output_field_key += "-" + output_typedef_num - message_output[output_field_key] = ( - outputs if len(outputs) > 1 else outputs[0] - ) - # success, return - return - except DecoderException as exc: - # this should be pretty common, don't be noisy or throw an exception - logging.debug( - "Could not decode a buffer for field number %s as a message: %s", - field_key, - exc, - ) - - # Decoding as a message did not work, try strings and then bytes - # The bytes decoding should never fail - for target_type in ["string", config.default_binary_type]: - try: - outputs = [] - decoder = blackboxprotobuf.lib.types.DECODERS[target_type] - for buf in buffers: - output, _ = decoder(buf, 0) - outputs.append(output) - - # all outputs worked, this is our type - # check if there is a message type already in the typedef - if "type" in field_typedef and "message" == field_typedef["type"]: - # we already had a message type. save it as an alt_typedef - - # check if we already have this type as an alt_typedef - output_typedef_nums = { - key: value - for key, value in field_typedef.setdefault( - "alt_typedefs", {} - ).items() - if value == target_type - }.keys() - output_typedef_num = None - if len(output_typedef_nums) == 0: - # find the next largest alt typedef number to put this type as - output_typedef_num = str( - max([int(i) for i in ["0"] + all_typedefs.keys()]) + 1 - ) - field_typedef.setdefault("alt_typedefs", {})[ - output_typedef_num - ] = target_type - else: - # we already have an alt typedef with this number - output_typedef_num = output_typedef_nums[0] - message_output[field_key + "-" + output_typedef_num] = ( - outputs if len(outputs) > 1 else outputs[0] - ) - else: - field_typedef["type"] = target_type - message_output[field_key] = outputs if len(outputs) > 1 else outputs[0] - return - except DecoderException: - continue - - -def encode_lendelim_message(data, config, typedef, path=None): - """Encode the length before the message""" - message_out = encode_message(data, config, typedef, path=path) - length = varint.encode_varint(len(message_out)) - logging.debug("Message length encoded: %d", len(length) + len(message_out)) - return length + message_out - - -def decode_lendelim_message(buf, config, typedef=None, pos=0, depth=0, path=None): - """Read in the length and use it as the end""" - length, pos = varint.decode_varint(buf, pos) - ret = decode_message( - buf, config, typedef, pos, pos + length, depth=depth, path=path - ) - return ret - - -def generate_packed_encoder(wrapped_encoder): - """Generate an encoder for a packed type from the base type encoder""" - - def length_wrapper(values): - """Encode repeat values and prefix with the length""" - output = bytearray() - for value in values: - output += wrapped_encoder(value) - length = varint.encode_varint(len(output)) - return length + output - - return length_wrapper - - -def generate_packed_decoder(wrapped_decoder): - """Generate an decoder for a packer type from a base type decoder""" - - def length_wrapper(buf, pos): - """Decode repeat values prefixed with the length""" - length, pos = varint.decode_varint(buf, pos) - end = pos + length - output = [] - while pos < end: - value, pos = wrapped_decoder(buf, pos) - output.append(value) - if pos > end: - raise DecoderException( - ( - "Error decoding packed field. Packed length larger than" - " buffer: decoded = %d, left = %d" - ) - % (length, len(buf) - pos) - ) - return output, pos - - return length_wrapper diff --git a/unfurl/lib/blackboxprotobuf/lib/types/type_maps.py b/unfurl/lib/blackboxprotobuf/lib/types/type_maps.py deleted file mode 100644 index b74d5d9..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/types/type_maps.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Contains various maps for protobuf types, including encoding/decoding - functions, wiretypes and default types -""" -from google.protobuf.internal import wire_format -from unfurl.lib.blackboxprotobuf.lib.types import fixed, length_delim -from unfurl.lib.blackboxprotobuf.lib.types import varint - -ENCODERS = { - "uint": varint.encode_uvarint, - "int": varint.encode_varint, - "sint": varint.encode_svarint, - "fixed32": fixed.encode_fixed32, - "sfixed32": fixed.encode_sfixed32, - "float": fixed.encode_float, - "fixed64": fixed.encode_fixed64, - "sfixed64": fixed.encode_sfixed64, - "double": fixed.encode_double, - "bytes": length_delim.encode_bytes, - "bytes_hex": length_delim.encode_bytes_hex, - "string": length_delim.encode_string, - "packed_uint": length_delim.generate_packed_encoder(varint.encode_uvarint), - "packed_int": length_delim.generate_packed_encoder(varint.encode_varint), - "packed_sint": length_delim.generate_packed_encoder(varint.encode_svarint), - "packed_fixed32": length_delim.generate_packed_encoder(fixed.encode_fixed32), - "packed_sfixed32": length_delim.generate_packed_encoder(fixed.encode_sfixed32), - "packed_float": length_delim.generate_packed_encoder(fixed.encode_float), - "packed_fixed64": length_delim.generate_packed_encoder(fixed.encode_fixed64), - "packed_sfixed64": length_delim.generate_packed_encoder(fixed.encode_sfixed64), - "packed_double": length_delim.generate_packed_encoder(fixed.encode_double), -} - -DECODERS = { - "uint": varint.decode_uvarint, - "int": varint.decode_varint, - "sint": varint.decode_svarint, - "fixed32": fixed.decode_fixed32, - "sfixed32": fixed.decode_sfixed32, - "float": fixed.decode_float, - "fixed64": fixed.decode_fixed64, - "sfixed64": fixed.decode_sfixed64, - "double": fixed.decode_double, - "bytes": length_delim.decode_bytes, - "bytes_hex": length_delim.decode_bytes_hex, - "string": length_delim.decode_string, - "packed_uint": length_delim.generate_packed_decoder(varint.decode_uvarint), - "packed_int": length_delim.generate_packed_decoder(varint.decode_varint), - "packed_sint": length_delim.generate_packed_decoder(varint.decode_svarint), - "packed_fixed32": length_delim.generate_packed_decoder(fixed.decode_fixed32), - "packed_sfixed32": length_delim.generate_packed_decoder(fixed.decode_sfixed32), - "packed_float": length_delim.generate_packed_decoder(fixed.decode_float), - "packed_fixed64": length_delim.generate_packed_decoder(fixed.decode_fixed64), - "packed_sfixed64": length_delim.generate_packed_decoder(fixed.decode_sfixed64), - "packed_double": length_delim.generate_packed_decoder(fixed.decode_double), -} - -WIRETYPES = { - "uint": wire_format.WIRETYPE_VARINT, - "int": wire_format.WIRETYPE_VARINT, - "sint": wire_format.WIRETYPE_VARINT, - "fixed32": wire_format.WIRETYPE_FIXED32, - "sfixed32": wire_format.WIRETYPE_FIXED32, - "float": wire_format.WIRETYPE_FIXED32, - "fixed64": wire_format.WIRETYPE_FIXED64, - "sfixed64": wire_format.WIRETYPE_FIXED64, - "double": wire_format.WIRETYPE_FIXED64, - "bytes": wire_format.WIRETYPE_LENGTH_DELIMITED, - "bytes_hex": wire_format.WIRETYPE_LENGTH_DELIMITED, - "string": wire_format.WIRETYPE_LENGTH_DELIMITED, - "message": wire_format.WIRETYPE_LENGTH_DELIMITED, - "group": wire_format.WIRETYPE_START_GROUP, - "packed_uint": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_int": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_sint": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_fixed32": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_sfixed32": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_float": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_fixed64": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_sfixed64": wire_format.WIRETYPE_LENGTH_DELIMITED, - "packed_double": wire_format.WIRETYPE_LENGTH_DELIMITED, -} - -WIRE_TYPE_DEFAULTS = { - wire_format.WIRETYPE_VARINT: "int", - wire_format.WIRETYPE_FIXED32: "fixed32", - wire_format.WIRETYPE_FIXED64: "fixed64", - wire_format.WIRETYPE_LENGTH_DELIMITED: None, - wire_format.WIRETYPE_START_GROUP: None, - wire_format.WIRETYPE_END_GROUP: None, -} diff --git a/unfurl/lib/blackboxprotobuf/lib/types/varint.py b/unfurl/lib/blackboxprotobuf/lib/types/varint.py deleted file mode 100644 index 45eaafb..0000000 --- a/unfurl/lib/blackboxprotobuf/lib/types/varint.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Classes for encoding and decoding varint types""" -import binascii -import struct - -from google.protobuf.internal import wire_format, encoder, decoder -import six - -from unfurl.lib.blackboxprotobuf.lib.exceptions import EncoderException, DecoderException - - -def _gen_append_bytearray(arr): - def _append_bytearray(x): - if isinstance(x, (str, int)): - arr.append(x) - elif isinstance(x, bytes): - arr.extend(x) - else: - raise EncoderException("Unknown type returned by protobuf library") - - return _append_bytearray - - -def encode_uvarint(value): - """Encode a long or int into a bytearray.""" - output = bytearray() - if value < 0: - raise EncoderException( - "Error encoding %d as uvarint. Value must be positive" % value - ) - try: - encoder._EncodeVarint(_gen_append_bytearray(output), value) - except (struct.error, ValueError) as exc: - six.raise_from(EncoderException("Error encoding %d as uvarint." % value), exc) - - return output - - -def decode_uvarint(buf, pos): - """Decode bytearray into a long.""" - # Convert buffer to string - if six.PY2: - buf = str(buf) - try: - value, pos = decoder._DecodeVarint(buf, pos) - except (TypeError, IndexError, decoder._DecodeError) as exc: - six.raise_from( - DecoderException( - "Error decoding uvarint from %s..." - % binascii.hexlify(buf[pos : pos + 8]) - ), - exc, - ) - return (value, pos) - - -def encode_varint(value): - """Encode a long or int into a bytearray.""" - output = bytearray() - if value > (2 ** 63) or value < -(2 ** 63): - raise EncoderException("Value %d above maximum varint size" % value) - try: - encoder._EncodeSignedVarint(_gen_append_bytearray(output), value) - except (struct.error, ValueError) as exc: - six.raise_from( - EncoderException("Error encoding %d as signed varint." % value), exc - ) - return output - - -def decode_varint(buf, pos): - """Decode bytearray into a long.""" - # Convert buffer to string - if six.PY2: - buf = str(buf) - try: - value, pos = decoder._DecodeSignedVarint(buf, pos) - except (TypeError, IndexError, decoder._DecodeError) as exc: - six.raise_from( - DecoderException( - "Error decoding varint from %s..." - % binascii.hexlify(buf[pos : pos + 8]) - ), - exc, - ) - return (value, pos) - - -def encode_svarint(value): - """Zigzag encode the potentially signed value prior to encoding""" - # zigzag encode value - return encode_uvarint(wire_format.ZigZagEncode(value)) - - -def decode_svarint(buf, pos): - """Decode bytearray into a long.""" - output, pos = decode_uvarint(buf, pos) - # zigzag encode value - return wire_format.ZigZagDecode(output), pos diff --git a/unfurl/parsers/parse_protobuf.py b/unfurl/parsers/parse_protobuf.py index 53df47a..bbc4561 100644 --- a/unfurl/parsers/parse_protobuf.py +++ b/unfurl/parsers/parse_protobuf.py @@ -13,7 +13,7 @@ # limitations under the License. import base64 -from unfurl.lib import blackboxprotobuf +import blackboxprotobuf from unfurl import utils proto_edge = { From 91f66cddf6460adbe6654e90e60b6634db242bb8 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 11 Oct 2024 11:06:57 -0700 Subject: [PATCH 2/2] Install requirements --- .github/workflows/unit-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5f25abd..2a857aa 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,7 +7,7 @@ on: pull_request: types: [opened, synchronize, reopened] push: - branches: master + branches: main jobs: build: @@ -29,7 +29,7 @@ jobs: python -m pip install --upgrade pip pip install pipenv pip install flake8 pytest - pip install -r requirements-all.txt + pip install -r requirements-all.txt -r requirements.txt - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names