Merge pull request #420 from hellohaptik/develop

Develop to Master 2021-03-08T09:36:00
hellohaptik · Mar 8, 2021 · 78ccecc · 78ccecc
2 parents 44b78ec + 20df637
commit 78ccecc
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 3 deletions.
diff --git a/ner_v2/detectors/numeral/number/standard_number_detector.py b/ner_v2/detectors/numeral/number/standard_number_detector.py
@@ -302,8 +302,12 @@ def _detect_number_from_digit(self, number_list=None, original_list=None):
                 scale = 1
 
             if number:
-                number = float(number) * scale
-                number = int(number) if number.is_integer() else number
+                if '.' not in number:
+                    number = int(number) * scale
+                else:
+                    number = float(number) * scale
+                    # FIXME: this conversion from float -> int is lossy, consider using Decimal class
+                    number = int(number) if number.is_integer() else number
                 unit = None
                 if self.unit_type:
                     unit, original_text = self._get_unit_from_text(original_text, processed_text)

diff --git a/ner_v2/detectors/numeral/utils.py b/ner_v2/detectors/numeral/utils.py
@@ -26,6 +26,7 @@ def get_number_from_number_word(text, number_word_dict):
         [In]  >> _get_number_from_numerals('two hundred three four hundred three',  number_word_dict)
         [Out] >> (['103', '403'], ['one hundred three', 'four hundred three'])
     """
+    # FIXME: conversion from float -> int is lossy, consider using Decimal class
     detected_number_list = []
     detected_original_text_list = []
 

diff --git a/ner_v2/tests/numeral/number/en/number_ner_tests.yaml b/ner_v2/tests/numeral/number/en/number_ner_tests.yaml
@@ -175,4 +175,18 @@ tests:
       - original_text: "50000 rupees"
         output_id: 3
         unit: rupees
-        value: "50000"
+        value: "50000"
+  - id: en_14
+    message: "here are some super large numbers 12345678901234567890 12345678 987654321012345678901 98765432101234567890123 "
+    unit_type: null
+    min_digit: 20
+    max_digit: 22
+    outputs:
+      - original_text: "12345678901234567890"
+        output_id: 1
+        unit: null
+        value: "12345678901234567890"
+      - original_text: "987654321012345678901"
+        output_id: 2
+        unit: null
+        value: "987654321012345678901"
diff --git a/ner_v2/tests/numeral/number/en/test_number_detection.py b/ner_v2/tests/numeral/number/en/test_number_detection.py
@@ -194,6 +194,9 @@ def run_test(self):
             message = testcase["message"]
             unit_type = testcase.get("unit_type", None)
             number_detector_object = NumberDetector(entity_name="number", language=language, unit_type=unit_type)
+            number_detector_object.set_min_max_digits(
+                min_digit=testcase.get('min_digit', number_detector_object.min_digit),
+                max_digit=testcase.get('max_digit', number_detector_object.max_digit))
             number_dicts, spans = number_detector_object.detect_entity(message)
 
             expected_number_dicts, expected_spans = parse_expected_outputs(testcase["outputs"])