From db5cfb49faa24ac2d1c8d10394084fbd035174e8 Mon Sep 17 00:00:00 2001 From: haptikrajasashtikar Date: Wed, 25 Oct 2023 13:43:21 +0530 Subject: [PATCH 1/5] Add logs for debugging the entity detection issue for NSDC --- ner_v2/api.py | 1 + ner_v2/detectors/textual/text_detection.py | 6 +++--- ner_v2/detectors/textual/utils.py | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ner_v2/api.py b/ner_v2/api.py index 965852a6..b52e40e4 100644 --- a/ner_v2/api.py +++ b/ner_v2/api.py @@ -799,6 +799,7 @@ def text(request): try: validate_text_request(request) + ner_logger.debug("Valid text request") # if verify success get detection data data = get_text_entity_detection_data(request) except InvalidTextRequest as err: diff --git a/ner_v2/detectors/textual/text_detection.py b/ner_v2/detectors/textual/text_detection.py index 34762367..73ee8044 100644 --- a/ner_v2/detectors/textual/text_detection.py +++ b/ner_v2/detectors/textual/text_detection.py @@ -442,7 +442,7 @@ def _get_bulk_text_detection_with_variants(self, messages): entity_list=entity_list, text=text, processed_text=processed_text) final_list.append(result_list) - + ner_logger.debug('[bulk_text_detection_with_variants] final_list:', final_list=final_list) return final_list def _get_entity_substring_from_text(self, text, variant, entity_name): @@ -668,7 +668,7 @@ def detect(self, message=None, **kwargs): entities[entity] = out data_list.append(entities) - + ner_logger.debug('[Single Message Detection] Final data', data_list=data_list) return data_list def detect_bulk(self, messages=None, **kwargs): @@ -768,7 +768,7 @@ def detect_bulk(self, messages=None, **kwargs): entities[entity] = out data_list.append(entities) - + ner_logger.debug('[Bulk Detection] Final data', data_list=data_list) return data_list @staticmethod diff --git a/ner_v2/detectors/textual/utils.py b/ner_v2/detectors/textual/utils.py index d305653d..7d219d1b 100644 --- a/ner_v2/detectors/textual/utils.py +++ b/ner_v2/detectors/textual/utils.py @@ -72,6 +72,7 @@ def get_detection(message, entity_dict, bot_message=None, language=ENGLISH_LANG, detected entity output """ + ner_logger.debug(f'[get_detection] message: {message}') text_detector = TextDetector(entity_dict=entity_dict, source_language_script=language, target_language_script=target_language_script) if isinstance(message, six.string_types): @@ -158,10 +159,12 @@ def get_text_entity_detection_data(request): """ request_data = json.loads(request.body) messages = request_data.get("messages", []) + ner_logger.debug(f"Request message data", entities=messages) bot_message = request_data.get("bot_message") entities = request_data.get("entities", {}) target_language_script = request_data.get('language_script') or ENGLISH_LANG source_language = request_data.get('source_language') or ENGLISH_LANG + ner_logger.debug(f"Request entity data", entities=entities) data = [] @@ -188,10 +191,12 @@ def get_text_entity_detection_data(request): # get detection for text entities which has ignore_message flag if fallback_value_entities: output = get_output_for_fallback_entities(fallback_value_entities, source_language) + ner_logger.debug('[output_for_fallback_entities] output: ', output=output) data[0]["entities"].update(output) # get detection for text entities if text_value_entities: + ner_logger.debug('[output_for_fallback_entities] text_value_entities exist') output = get_detection(message=message_str, entity_dict=text_value_entities, structured_value=None, bot_message=bot_message, language_script=source_language, @@ -202,14 +207,13 @@ def get_text_entity_detection_data(request): elif len(messages) > 1: text_detection_result = get_detection(message=messages, entity_dict=entities, structured_value=None, bot_message=bot_message) - data = [{"entities": x, "language": source_language} for x in text_detection_result] else: ner_logger.debug("No valid message provided") raise InvalidTextRequest(f"Key `messages` is required to be a non-empty List[str], " f"but got a list with length {message_len}") - + ner_logger.debug(f"Final data", data=data) return data From 36e96a822fe07882e106c6b2eec0ae326025897a Mon Sep 17 00:00:00 2001 From: haptikrajasashtikar Date: Wed, 25 Oct 2023 14:50:34 +0530 Subject: [PATCH 2/5] Update a key in logging --- ner_v2/detectors/textual/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ner_v2/detectors/textual/utils.py b/ner_v2/detectors/textual/utils.py index 7d219d1b..db6c6508 100644 --- a/ner_v2/detectors/textual/utils.py +++ b/ner_v2/detectors/textual/utils.py @@ -159,7 +159,7 @@ def get_text_entity_detection_data(request): """ request_data = json.loads(request.body) messages = request_data.get("messages", []) - ner_logger.debug(f"Request message data", entities=messages) + ner_logger.debug(f"Request message data", messages=messages) bot_message = request_data.get("bot_message") entities = request_data.get("entities", {}) target_language_script = request_data.get('language_script') or ENGLISH_LANG From c4d379d60a2e6e3e8171a1f1b596caa80923ddb3 Mon Sep 17 00:00:00 2001 From: haptikrajasashtikar Date: Wed, 25 Oct 2023 17:36:15 +0530 Subject: [PATCH 3/5] Update the loggers --- ner_v2/detectors/textual/text_detection.py | 2 -- ner_v2/detectors/textual/utils.py | 12 +++++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ner_v2/detectors/textual/text_detection.py b/ner_v2/detectors/textual/text_detection.py index 73ee8044..ef930fd2 100644 --- a/ner_v2/detectors/textual/text_detection.py +++ b/ner_v2/detectors/textual/text_detection.py @@ -668,7 +668,6 @@ def detect(self, message=None, **kwargs): entities[entity] = out data_list.append(entities) - ner_logger.debug('[Single Message Detection] Final data', data_list=data_list) return data_list def detect_bulk(self, messages=None, **kwargs): @@ -768,7 +767,6 @@ def detect_bulk(self, messages=None, **kwargs): entities[entity] = out data_list.append(entities) - ner_logger.debug('[Bulk Detection] Final data', data_list=data_list) return data_list @staticmethod diff --git a/ner_v2/detectors/textual/utils.py b/ner_v2/detectors/textual/utils.py index db6c6508..97b5d620 100644 --- a/ner_v2/detectors/textual/utils.py +++ b/ner_v2/detectors/textual/utils.py @@ -78,8 +78,10 @@ def get_detection(message, entity_dict, bot_message=None, language=ENGLISH_LANG, if isinstance(message, six.string_types): entity_output = text_detector.detect(message=message, bot_message=bot_message) + ner_logger.debug(f'[Single Message Detection] Entity Output {entity_output}') elif isinstance(message, (list, tuple)): entity_output = text_detector.detect_bulk(messages=message) + ner_logger.debug(f'[Multiple Message Detection] Entity Output {entity_output}') else: raise TypeError('`message` argument must be either of type `str`, `unicode`, `list` or `tuple`.') @@ -159,12 +161,12 @@ def get_text_entity_detection_data(request): """ request_data = json.loads(request.body) messages = request_data.get("messages", []) - ner_logger.debug(f"Request message data", messages=messages) + ner_logger.debug(f"Request message data {messages}") bot_message = request_data.get("bot_message") entities = request_data.get("entities", {}) target_language_script = request_data.get('language_script') or ENGLISH_LANG source_language = request_data.get('source_language') or ENGLISH_LANG - ner_logger.debug(f"Request entity data", entities=entities) + ner_logger.debug(f"Request entity data {entities}") data = [] @@ -191,12 +193,12 @@ def get_text_entity_detection_data(request): # get detection for text entities which has ignore_message flag if fallback_value_entities: output = get_output_for_fallback_entities(fallback_value_entities, source_language) - ner_logger.debug('[output_for_fallback_entities] output: ', output=output) + ner_logger.debug(f'[output_for_fallback_entities] output: {output}') data[0]["entities"].update(output) # get detection for text entities if text_value_entities: - ner_logger.debug('[output_for_fallback_entities] text_value_entities exist') + ner_logger.debug(f'[output_for_fallback_entities] text_value_entities {text_value_entities}') output = get_detection(message=message_str, entity_dict=text_value_entities, structured_value=None, bot_message=bot_message, language_script=source_language, @@ -213,7 +215,7 @@ def get_text_entity_detection_data(request): ner_logger.debug("No valid message provided") raise InvalidTextRequest(f"Key `messages` is required to be a non-empty List[str], " f"but got a list with length {message_len}") - ner_logger.debug(f"Final data", data=data) + ner_logger.debug(f"Final data {data}") return data From d90f71dd15da2c8e74c93f0019afd70749cca6ac Mon Sep 17 00:00:00 2001 From: haptikrajasashtikar Date: Wed, 25 Oct 2023 17:40:03 +0530 Subject: [PATCH 4/5] Update the loggers 2 --- ner_v2/detectors/textual/text_detection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ner_v2/detectors/textual/text_detection.py b/ner_v2/detectors/textual/text_detection.py index ef930fd2..68e1c76d 100644 --- a/ner_v2/detectors/textual/text_detection.py +++ b/ner_v2/detectors/textual/text_detection.py @@ -442,7 +442,7 @@ def _get_bulk_text_detection_with_variants(self, messages): entity_list=entity_list, text=text, processed_text=processed_text) final_list.append(result_list) - ner_logger.debug('[bulk_text_detection_with_variants] final_list:', final_list=final_list) + ner_logger.debug(f'[bulk_text_detection_with_variants] final_list: {final_list}') return final_list def _get_entity_substring_from_text(self, text, variant, entity_name): From 56fb55d8e219c60f5499036301c5fb786a0da17d Mon Sep 17 00:00:00 2001 From: haptikrajasashtikar Date: Thu, 26 Oct 2023 14:42:42 +0530 Subject: [PATCH 5/5] Update the version of NodeJS in the Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f43808f..a0d9f87a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ COPY docker/nginx.conf /etc/nginx/nginx.conf COPY docker/default.site.conf /etc/nginx/sites-available/default # TODO: Separate this out to a dev/test docker image -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash && \ +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash && \ apt-get install nodejs && \ npm install -g newman && \ npm install -g newman-reporter-htmlextra && \