From e6133b0456e47166f74ac07b7d75ac413dd03e3f Mon Sep 17 00:00:00 2001 From: Raza Sayed Date: Wed, 19 Feb 2020 09:59:36 +0000 Subject: [PATCH 1/4] Use absolute import --- chatbot_ner/config.py | 1 + chatbot_ner/settings.py | 1 + chatbot_ner/wsgi.py | 2 +- datastore/commands/create_datastore.py | 1 + datastore/commands/delete_datastore.py | 1 + datastore/commands/delete_entity_data_datastore.py | 1 + datastore/commands/populate_datastore.py | 3 +-- datastore/commands/repopulate_datastore.py | 3 +-- datastore/constants.py | 1 + datastore/elastic_search/__init__.py | 1 + datastore/elastic_search/connect.py | 1 + datastore/elastic_search/transfer.py | 1 + datastore/utils.py | 1 + external_api/api.py | 1 + initial_setup.py | 2 +- lib/aws_utils.py | 1 + lib/nlp/const.py | 1 + lib/nlp/etc.py | 1 + lib/nlp/lemmatizer.py | 1 + lib/nlp/ngram.py | 3 ++- lib/nlp/pos.py | 1 + lib/nlp/regexreplace.py | 1 + lib/nlp/stemmer.py | 1 + lib/nlp/tokenizer.py | 2 +- manage.py | 1 + models/crf/models.py | 1 + models/crf/output_generation/city.py | 1 + models/crf/output_generation/date.py | 1 + models/crf/test.py | 1 + models/crf_v2/crf_detect_entity.py | 1 + models/crf_v2/crf_preprocess_data.py | 1 + models/crf_v2/crf_train.py | 1 + models/crf_v2/get_crf_tagger.py | 1 + models/crf_v2/load_word_embeddings.py | 1 + ner_v1/chatbot/combine_detection_logic.py | 3 +-- ner_v1/chatbot/tag_message.py | 1 + ner_v1/detectors/numeral/budget/budget_detection.py | 2 +- ner_v1/detectors/numeral/number/number_detection.py | 3 +-- ner_v1/detectors/numeral/number/passenger_detection.py | 2 +- ner_v1/detectors/numeral/size/shopping_size_detection.py | 2 +- ner_v1/detectors/pattern/email/email_detection.py | 3 +-- ner_v1/detectors/pattern/phone_number/phone_detection.py | 3 +-- ner_v1/detectors/pattern/pnr/pnr_detection.py | 2 +- ner_v1/detectors/pattern/regex/regex_detection.py | 1 + ner_v1/detectors/temporal/date/date_detection.py | 1 + ner_v1/detectors/temporal/time/time_detection.py | 1 + ner_v1/detectors/textual/city/city_detection.py | 1 + ner_v1/detectors/textual/name/name_detection.py | 1 + ner_v1/detectors/textual/text/text_detection.py | 1 + ner_v2/api.py | 1 + ner_v2/detectors/numeral/number/en/number_detection.py | 1 + ner_v2/detectors/numeral/number/number_detection.py | 1 + ner_v2/detectors/numeral/number/standard_number_detector.py | 1 + .../numeral/number_range/standard_number_range_detector.py | 1 - ner_v2/detectors/numeral/utils.py | 1 + .../detectors/pattern/phone_number/phone_number_detection.py | 1 + ner_v2/detectors/temporal/date/en/date_detection.py | 1 + ner_v2/detectors/temporal/date/hi/date_detection.py | 1 + ner_v2/detectors/temporal/date/standard_date_regex.py | 2 +- ner_v2/detectors/temporal/time/en/time_detection.py | 1 + ner_v2/detectors/temporal/time/time_detection.py | 1 + ner_v2/detectors/temporal/utils.py | 1 + ner_v2/detectors/utils.py | 1 + 63 files changed, 63 insertions(+), 22 deletions(-) diff --git a/chatbot_ner/config.py b/chatbot_ner/config.py index f1f0e9d0a..1bbbb9aa3 100644 --- a/chatbot_ner/config.py +++ b/chatbot_ner/config.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import logging.handlers import os diff --git a/chatbot_ner/settings.py b/chatbot_ner/settings.py index de8e91654..3da923452 100755 --- a/chatbot_ner/settings.py +++ b/chatbot_ner/settings.py @@ -9,6 +9,7 @@ """ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) +from __future__ import absolute_import import os import sys diff --git a/chatbot_ner/wsgi.py b/chatbot_ner/wsgi.py index 9d1353e5d..413a2b682 100755 --- a/chatbot_ner/wsgi.py +++ b/chatbot_ner/wsgi.py @@ -6,7 +6,7 @@ For more information on this file, see https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ """ - +from __future__ import absolute_import import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "chatbot_ner.settings") diff --git a/datastore/commands/create_datastore.py b/datastore/commands/create_datastore.py index a5ae20a38..733d34b3b 100644 --- a/datastore/commands/create_datastore.py +++ b/datastore/commands/create_datastore.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from django.core.management.base import BaseCommand from datastore import DataStore diff --git a/datastore/commands/delete_datastore.py b/datastore/commands/delete_datastore.py index 564a306aa..2cee6c248 100644 --- a/datastore/commands/delete_datastore.py +++ b/datastore/commands/delete_datastore.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from django.core.management.base import BaseCommand from datastore import DataStore diff --git a/datastore/commands/delete_entity_data_datastore.py b/datastore/commands/delete_entity_data_datastore.py index b089b6e3e..2ddbfee6a 100644 --- a/datastore/commands/delete_entity_data_datastore.py +++ b/datastore/commands/delete_entity_data_datastore.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from django.core.management.base import BaseCommand from datastore import DataStore diff --git a/datastore/commands/populate_datastore.py b/datastore/commands/populate_datastore.py index c750faea6..ef6e14c01 100644 --- a/datastore/commands/populate_datastore.py +++ b/datastore/commands/populate_datastore.py @@ -1,7 +1,6 @@ +from __future__ import absolute_import import os - from django.core.management.base import BaseCommand - from datastore import DataStore diff --git a/datastore/commands/repopulate_datastore.py b/datastore/commands/repopulate_datastore.py index 2f778d291..ce1fafa51 100644 --- a/datastore/commands/repopulate_datastore.py +++ b/datastore/commands/repopulate_datastore.py @@ -1,7 +1,6 @@ +from __future__ import absolute_import import os - from django.core.management.base import BaseCommand - from datastore import DataStore diff --git a/datastore/constants.py b/datastore/constants.py index f7f76da19..fba25dbd6 100644 --- a/datastore/constants.py +++ b/datastore/constants.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import elasticsearch import os from chatbot_ner.settings import BASE_DIR diff --git a/datastore/elastic_search/__init__.py b/datastore/elastic_search/__init__.py index 5618d0c87..fec2af426 100644 --- a/datastore/elastic_search/__init__.py +++ b/datastore/elastic_search/__init__.py @@ -1 +1,2 @@ +from __future__ import absolute_import from . import connect, create, populate, query, transfer diff --git a/datastore/elastic_search/connect.py b/datastore/elastic_search/connect.py index fd7469040..c3c1478a7 100644 --- a/datastore/elastic_search/connect.py +++ b/datastore/elastic_search/connect.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from elasticsearch import Elasticsearch from chatbot_ner.config import CHATBOT_NER_DATASTORE from datastore.elastic_search.transfer import ESTransfer diff --git a/datastore/elastic_search/transfer.py b/datastore/elastic_search/transfer.py index 504a430ca..092865ef2 100644 --- a/datastore/elastic_search/transfer.py +++ b/datastore/elastic_search/transfer.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import requests import json from elasticsearch import Elasticsearch, RequestsHttpConnection diff --git a/datastore/utils.py b/datastore/utils.py index 7ed54468c..c66cffd55 100644 --- a/datastore/utils.py +++ b/datastore/utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import csv import os from collections import defaultdict diff --git a/external_api/api.py b/external_api/api.py index 4a96b2e7b..f486f212d 100644 --- a/external_api/api.py +++ b/external_api/api.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import json from django.http import HttpResponse from datastore.datastore import DataStore diff --git a/initial_setup.py b/initial_setup.py index 6f41fc62f..ba335c309 100755 --- a/initial_setup.py +++ b/initial_setup.py @@ -1,6 +1,6 @@ +from __future__ import absolute_import import os import time - import nltk BASE_DIR = os.path.dirname(__file__) diff --git a/lib/aws_utils.py b/lib/aws_utils.py index c38029859..c36446565 100644 --- a/lib/aws_utils.py +++ b/lib/aws_utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import boto import boto3 from chatbot_ner.config import ner_logger diff --git a/lib/nlp/const.py b/lib/nlp/const.py index 0d522499b..7c246621b 100644 --- a/lib/nlp/const.py +++ b/lib/nlp/const.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import os from lib.nlp.etc import store_data_in_list from lib.nlp.ngram import Ngram diff --git a/lib/nlp/etc.py b/lib/nlp/etc.py index dc0c8ca02..ba8141ac0 100644 --- a/lib/nlp/etc.py +++ b/lib/nlp/etc.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import csv diff --git a/lib/nlp/lemmatizer.py b/lib/nlp/lemmatizer.py index e7fd09f36..605916f02 100644 --- a/lib/nlp/lemmatizer.py +++ b/lib/nlp/lemmatizer.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from nltk.stem import WordNetLemmatizer # constants diff --git a/lib/nlp/ngram.py b/lib/nlp/ngram.py index 6f8f388ae..1b76784d0 100644 --- a/lib/nlp/ngram.py +++ b/lib/nlp/ngram.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import nltk @@ -39,7 +40,7 @@ def ngram_list(n, word_list, stop_word_list=None): all_ngrams = nltk.ngrams(word_list, n) ngram_list = [] for ngram in all_ngrams: - lowered_ngram_tokens = map(lambda token: token.lower(), ngram) + lowered_ngram_tokens = [token.lower() for token in ngram] if any(token not in stop_word_set for token in lowered_ngram_tokens): ngram_list.append(' '.join(ngram)) return ngram_list diff --git a/lib/nlp/pos.py b/lib/nlp/pos.py index bff058951..4c5471a9a 100644 --- a/lib/nlp/pos.py +++ b/lib/nlp/pos.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import nltk # constants diff --git a/lib/nlp/regexreplace.py b/lib/nlp/regexreplace.py index 3bd03783b..399d264d0 100644 --- a/lib/nlp/regexreplace.py +++ b/lib/nlp/regexreplace.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import re diff --git a/lib/nlp/stemmer.py b/lib/nlp/stemmer.py index 74a3305b7..9adcfd3c7 100644 --- a/lib/nlp/stemmer.py +++ b/lib/nlp/stemmer.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from nltk.stem.porter import PorterStemmer # constants diff --git a/lib/nlp/tokenizer.py b/lib/nlp/tokenizer.py index b02e39cd7..df29d6e5e 100644 --- a/lib/nlp/tokenizer.py +++ b/lib/nlp/tokenizer.py @@ -1,5 +1,5 @@ +from __future__ import absolute_import import re - import nltk import regex diff --git a/manage.py b/manage.py index 7aefcc5e4..96190a83d 100755 --- a/manage.py +++ b/manage.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import absolute_import import os import sys diff --git a/models/crf/models.py b/models/crf/models.py index 95e7c31f2..292eaeda9 100644 --- a/models/crf/models.py +++ b/models/crf/models.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from chatbot_ner.config import CITY_MODEL_TYPE, DATE_MODEL_TYPE from .constant import CRF_MODEL_TYPE, CITY_ENTITY_TYPE, DATE_ENTITY_TYPE from .test import PredictCRF diff --git a/models/crf/output_generation/city.py b/models/crf/output_generation/city.py index 595490092..db71b48ec 100644 --- a/models/crf/output_generation/city.py +++ b/models/crf/output_generation/city.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import models.crf.constant as model_constant diff --git a/models/crf/output_generation/date.py b/models/crf/output_generation/date.py index 1fa2c9baf..fc847768b 100644 --- a/models/crf/output_generation/date.py +++ b/models/crf/output_generation/date.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import models.crf.constant as model_constant diff --git a/models/crf/test.py b/models/crf/test.py index 443441318..223d65154 100644 --- a/models/crf/test.py +++ b/models/crf/test.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import from chatbot_ner.config import ner_logger, CITY_MODEL_PATH, DATE_MODEL_PATH from lib.nlp.const import nltk_tokenizer from lib.nlp.pos import POS diff --git a/models/crf_v2/crf_detect_entity.py b/models/crf_v2/crf_detect_entity.py index 94eeeb886..9d9836b16 100644 --- a/models/crf_v2/crf_detect_entity.py +++ b/models/crf_v2/crf_detect_entity.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from lib.nlp.tokenizer import Tokenizer, NLTK_TOKENIZER from .crf_preprocess_data import CrfPreprocessData from .get_crf_tagger import CrfModel diff --git a/models/crf_v2/crf_preprocess_data.py b/models/crf_v2/crf_preprocess_data.py index cb7bed269..870557c3d 100644 --- a/models/crf_v2/crf_preprocess_data.py +++ b/models/crf_v2/crf_preprocess_data.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import numpy as np from lib.nlp.pos import POS import re diff --git a/models/crf_v2/crf_train.py b/models/crf_v2/crf_train.py index aee7a9cf8..49a44e555 100644 --- a/models/crf_v2/crf_train.py +++ b/models/crf_v2/crf_train.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import pycrfsuite from chatbot_ner.config import ner_logger, CRF_MODEL_S3_BUCKET_NAME, CRF_MODEL_S3_BUCKET_REGION, CRF_MODELS_PATH from datastore.datastore import DataStore diff --git a/models/crf_v2/get_crf_tagger.py b/models/crf_v2/get_crf_tagger.py index 4b774f9d6..dd661adca 100644 --- a/models/crf_v2/get_crf_tagger.py +++ b/models/crf_v2/get_crf_tagger.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from lib.singleton import Singleton from chatbot_ner.config import CRF_MODEL_S3_BUCKET_REGION, CRF_MODEL_S3_BUCKET_NAME, ner_logger from lib.aws_utils import read_model_dict_from_s3 diff --git a/models/crf_v2/load_word_embeddings.py b/models/crf_v2/load_word_embeddings.py index f603fc82d..0916af853 100644 --- a/models/crf_v2/load_word_embeddings.py +++ b/models/crf_v2/load_word_embeddings.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import numpy as np from lib.singleton import Singleton import pickle diff --git a/ner_v1/chatbot/combine_detection_logic.py b/ner_v1/chatbot/combine_detection_logic.py index 2eb6f08c2..9c23df252 100644 --- a/ner_v1/chatbot/combine_detection_logic.py +++ b/ner_v1/chatbot/combine_detection_logic.py @@ -1,7 +1,6 @@ +from __future__ import absolute_import from collections import defaultdict - from six import iteritems - from lib.nlp.const import TOKENIZER from lib.nlp.regexreplace import RegexReplace from ner_constants import ORIGINAL_TEXT, DETECTION_METHOD, FROM_MESSAGE, FROM_MODEL_VERIFIED, FROM_MODEL_NOT_VERIFIED diff --git a/ner_v1/chatbot/tag_message.py b/ner_v1/chatbot/tag_message.py index 395085f25..b2ff7e3f3 100644 --- a/ner_v1/chatbot/tag_message.py +++ b/ner_v1/chatbot/tag_message.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from ner_v1.chatbot.combine_detection_logic import combine_output_of_detection_logic_and_tag from ner_v1.chatbot.entity_detection import get_text, get_city, get_date, get_time, get_email, \ get_phone_number, get_budget, get_number, get_pnr, get_shopping_size diff --git a/ner_v1/detectors/numeral/budget/budget_detection.py b/ner_v1/detectors/numeral/budget/budget_detection.py index ab2dd153d..c817a6211 100644 --- a/ner_v1/detectors/numeral/budget/budget_detection.py +++ b/ner_v1/detectors/numeral/budget/budget_detection.py @@ -1,5 +1,5 @@ +from __future__ import absolute_import import re - from language_utilities.constant import ENGLISH_LANG from ner_v1.constant import BUDGET_TYPE_NORMAL, BUDGET_TYPE_TEXT from ner_v1.detectors.base_detector import BaseDetector diff --git a/ner_v1/detectors/numeral/number/number_detection.py b/ner_v1/detectors/numeral/number/number_detection.py index 03d9e8424..6dec1b830 100644 --- a/ner_v1/detectors/numeral/number/number_detection.py +++ b/ner_v1/detectors/numeral/number/number_detection.py @@ -1,7 +1,6 @@ +from __future__ import absolute_import import re - from word2number import w2n - from lib.nlp.const import nltk_tokenizer from ner_v1.constant import DIGIT_UNITS, NUMERIC_VARIANTS from ner_v1.detectors.base_detector import BaseDetector diff --git a/ner_v1/detectors/numeral/number/passenger_detection.py b/ner_v1/detectors/numeral/number/passenger_detection.py index 50ac37970..307595868 100644 --- a/ner_v1/detectors/numeral/number/passenger_detection.py +++ b/ner_v1/detectors/numeral/number/passenger_detection.py @@ -1,5 +1,5 @@ +from __future__ import absolute_import import re - from ner_v1.detectors.base_detector import BaseDetector from ner_v1.detectors.numeral.number.number_detection import NumberDetector from language_utilities.constant import ENGLISH_LANG diff --git a/ner_v1/detectors/numeral/size/shopping_size_detection.py b/ner_v1/detectors/numeral/size/shopping_size_detection.py index f4c71b957..732a262e7 100644 --- a/ner_v1/detectors/numeral/size/shopping_size_detection.py +++ b/ner_v1/detectors/numeral/size/shopping_size_detection.py @@ -1,5 +1,5 @@ +from __future__ import absolute_import import re - from ner_v1.detectors.textual.text.text_detection import TextDetector from ner_v1.detectors.base_detector import BaseDetector from language_utilities.constant import ENGLISH_LANG diff --git a/ner_v1/detectors/pattern/email/email_detection.py b/ner_v1/detectors/pattern/email/email_detection.py index 5cf62a018..73aa3fe05 100644 --- a/ner_v1/detectors/pattern/email/email_detection.py +++ b/ner_v1/detectors/pattern/email/email_detection.py @@ -1,6 +1,5 @@ +from __future__ import absolute_import import re - - from ner_v1.detectors.base_detector import BaseDetector from language_utilities.constant import ENGLISH_LANG diff --git a/ner_v1/detectors/pattern/phone_number/phone_detection.py b/ner_v1/detectors/pattern/phone_number/phone_detection.py index 794d503f6..1065aa966 100644 --- a/ner_v1/detectors/pattern/phone_number/phone_detection.py +++ b/ner_v1/detectors/pattern/phone_number/phone_detection.py @@ -1,6 +1,5 @@ +from __future__ import absolute_import import re - - from ner_v1.detectors.base_detector import BaseDetector from language_utilities.constant import ENGLISH_LANG diff --git a/ner_v1/detectors/pattern/pnr/pnr_detection.py b/ner_v1/detectors/pattern/pnr/pnr_detection.py index b0f33a015..a5aca1715 100644 --- a/ner_v1/detectors/pattern/pnr/pnr_detection.py +++ b/ner_v1/detectors/pattern/pnr/pnr_detection.py @@ -1,5 +1,5 @@ +from __future__ import absolute_import import re - from ner_v1.detectors.base_detector import BaseDetector from language_utilities.constant import ENGLISH_LANG diff --git a/ner_v1/detectors/pattern/regex/regex_detection.py b/ner_v1/detectors/pattern/regex/regex_detection.py index 954a4db4d..57ebd8955 100644 --- a/ner_v1/detectors/pattern/regex/regex_detection.py +++ b/ner_v1/detectors/pattern/regex/regex_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import re from chatbot_ner.config import ner_logger diff --git a/ner_v1/detectors/temporal/date/date_detection.py b/ner_v1/detectors/temporal/date/date_detection.py index 1b7904ecb..65b38f0ab 100644 --- a/ner_v1/detectors/temporal/date/date_detection.py +++ b/ner_v1/detectors/temporal/date/date_detection.py @@ -1,4 +1,5 @@ # coding=utf-8 +from __future__ import absolute_import import copy import datetime import re diff --git a/ner_v1/detectors/temporal/time/time_detection.py b/ner_v1/detectors/temporal/time/time_detection.py index f094a9624..22b3df952 100644 --- a/ner_v1/detectors/temporal/time/time_detection.py +++ b/ner_v1/detectors/temporal/time/time_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from datetime import datetime import re import pytz diff --git a/ner_v1/detectors/textual/city/city_detection.py b/ner_v1/detectors/textual/city/city_detection.py index b9cbec450..ae6aabdcc 100644 --- a/ner_v1/detectors/textual/city/city_detection.py +++ b/ner_v1/detectors/textual/city/city_detection.py @@ -1,4 +1,5 @@ # coding=utf-8 +from __future__ import absolute_import import re import models.crf.constant as model_constant diff --git a/ner_v1/detectors/textual/name/name_detection.py b/ner_v1/detectors/textual/name/name_detection.py index 852eba85e..317d2cb91 100644 --- a/ner_v1/detectors/textual/name/name_detection.py +++ b/ner_v1/detectors/textual/name/name_detection.py @@ -1,4 +1,5 @@ # coding=utf-8 +from __future__ import absolute_import import re import string diff --git a/ner_v1/detectors/textual/text/text_detection.py b/ner_v1/detectors/textual/text/text_detection.py index a6e214317..3b1d5aecc 100644 --- a/ner_v1/detectors/textual/text/text_detection.py +++ b/ner_v1/detectors/textual/text/text_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import collections import string diff --git a/ner_v2/api.py b/ner_v2/api.py index 69f12ac6b..611662a4c 100644 --- a/ner_v2/api.py +++ b/ner_v2/api.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import from chatbot_ner.config import ner_logger from ner_constants import PARAMETER_MESSAGE, PARAMETER_ENTITY_NAME, PARAMETER_STRUCTURED_VALUE, \ PARAMETER_FALLBACK_VALUE, \ diff --git a/ner_v2/detectors/numeral/number/en/number_detection.py b/ner_v2/detectors/numeral/number/en/number_detection.py index a02aeb4bc..c3209f0b3 100644 --- a/ner_v2/detectors/numeral/number/en/number_detection.py +++ b/ner_v2/detectors/numeral/number/en/number_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import re import os diff --git a/ner_v2/detectors/numeral/number/number_detection.py b/ner_v2/detectors/numeral/number/number_detection.py index e4f7651ac..050f7f062 100644 --- a/ner_v2/detectors/numeral/number/number_detection.py +++ b/ner_v2/detectors/numeral/number/number_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import importlib import math import os diff --git a/ner_v2/detectors/numeral/number/standard_number_detector.py b/ner_v2/detectors/numeral/number/standard_number_detector.py index bbb0f5fd7..b7e86b463 100644 --- a/ner_v2/detectors/numeral/number/standard_number_detector.py +++ b/ner_v2/detectors/numeral/number/standard_number_detector.py @@ -1,4 +1,5 @@ # coding=utf-8 +from __future__ import absolute_import import pandas as pd import collections import os diff --git a/ner_v2/detectors/numeral/number_range/standard_number_range_detector.py b/ner_v2/detectors/numeral/number_range/standard_number_range_detector.py index 847dbbb32..e15836851 100644 --- a/ner_v2/detectors/numeral/number_range/standard_number_range_detector.py +++ b/ner_v2/detectors/numeral/number_range/standard_number_range_detector.py @@ -1,6 +1,5 @@ # coding=utf-8 from __future__ import absolute_import - import pandas as pd import collections import os diff --git a/ner_v2/detectors/numeral/utils.py b/ner_v2/detectors/numeral/utils.py index 61d4f055c..573605b05 100644 --- a/ner_v2/detectors/numeral/utils.py +++ b/ner_v2/detectors/numeral/utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import re from six.moves import range diff --git a/ner_v2/detectors/pattern/phone_number/phone_number_detection.py b/ner_v2/detectors/pattern/phone_number/phone_number_detection.py index 6e55fe254..9b39ce090 100644 --- a/ner_v2/detectors/pattern/phone_number/phone_number_detection.py +++ b/ner_v2/detectors/pattern/phone_number/phone_number_detection.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import from ner_v2.detectors.base_detector import BaseDetector from ner_v2.detectors.numeral.number.number_detection import NumberDetector from language_utilities.constant import ENGLISH_LANG diff --git a/ner_v2/detectors/temporal/date/en/date_detection.py b/ner_v2/detectors/temporal/date/en/date_detection.py index b65afd67e..48e91504d 100644 --- a/ner_v2/detectors/temporal/date/en/date_detection.py +++ b/ner_v2/detectors/temporal/date/en/date_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import copy import datetime import re diff --git a/ner_v2/detectors/temporal/date/hi/date_detection.py b/ner_v2/detectors/temporal/date/hi/date_detection.py index 9bfbd7b0d..9b575ec85 100644 --- a/ner_v2/detectors/temporal/date/hi/date_detection.py +++ b/ner_v2/detectors/temporal/date/hi/date_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import re import os diff --git a/ner_v2/detectors/temporal/date/standard_date_regex.py b/ner_v2/detectors/temporal/date/standard_date_regex.py index 44adbd7c6..e5f726caf 100644 --- a/ner_v2/detectors/temporal/date/standard_date_regex.py +++ b/ner_v2/detectors/temporal/date/standard_date_regex.py @@ -1,5 +1,5 @@ # coding=utf-8 - +from __future__ import absolute_import import datetime import re from dateutil.relativedelta import relativedelta diff --git a/ner_v2/detectors/temporal/time/en/time_detection.py b/ner_v2/detectors/temporal/time/en/time_detection.py index 5a94f8c65..1fc1fe4eb 100644 --- a/ner_v2/detectors/temporal/time/en/time_detection.py +++ b/ner_v2/detectors/temporal/time/en/time_detection.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import re import datetime import collections diff --git a/ner_v2/detectors/temporal/time/time_detection.py b/ner_v2/detectors/temporal/time/time_detection.py index d85e708ec..4091662bb 100644 --- a/ner_v2/detectors/temporal/time/time_detection.py +++ b/ner_v2/detectors/temporal/time/time_detection.py @@ -1,4 +1,5 @@ # coding=utf-8 +from __future__ import absolute_import import importlib import os diff --git a/ner_v2/detectors/temporal/utils.py b/ner_v2/detectors/temporal/utils.py index 80f6d308f..f314318c5 100644 --- a/ner_v2/detectors/temporal/utils.py +++ b/ner_v2/detectors/temporal/utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import import calendar from datetime import datetime, timedelta, tzinfo # FIXME: Change import to `import datetime` diff --git a/ner_v2/detectors/utils.py b/ner_v2/detectors/utils.py index c20325eef..85b13ef73 100644 --- a/ner_v2/detectors/utils.py +++ b/ner_v2/detectors/utils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from ner_v2.constant import LANGUAGE_DATA_DIRECTORY import os From ab2a9bd816a8fbceeacc798fbba2c68aadc77146 Mon Sep 17 00:00:00 2001 From: Raza Sayed Date: Wed, 19 Feb 2020 11:16:03 +0000 Subject: [PATCH 2/4] Changes to attach a metaclass --- datastore/datastore.py | 4 ++-- lib/nlp/lemmatizer.py | 5 ++--- lib/nlp/pos.py | 4 ++-- lib/nlp/stemmer.py | 5 ++--- lib/nlp/tokenizer.py | 5 ++--- models/crf_v2/get_crf_tagger.py | 4 ++-- models/crf_v2/load_word_embeddings.py | 4 ++-- ner_v1/detectors/base_detector.py | 3 +-- 8 files changed, 15 insertions(+), 19 deletions(-) diff --git a/datastore/datastore.py b/datastore/datastore.py index 270dfe73f..5a95b8a5d 100644 --- a/datastore/datastore.py +++ b/datastore/datastore.py @@ -10,9 +10,10 @@ from datastore.exceptions import (DataStoreSettingsImproperlyConfiguredException, EngineNotImplementedException, EngineConnectionException, NonESEngineTransferException, IndexNotFoundException) from lib.singleton import Singleton +import six -class DataStore(object): +class DataStore(six.with_metaclass(Singleton, object)): """ Singleton class to connect to engine storing entity related data @@ -34,7 +35,6 @@ class DataStore(object): _store_name: Name of the database/index to query on the engine server _client_or_connection: Low level connection object to the engine, None at initialization """ - __metaclass__ = Singleton def __init__(self): """ diff --git a/lib/nlp/lemmatizer.py b/lib/nlp/lemmatizer.py index 605916f02..a71b4b148 100644 --- a/lib/nlp/lemmatizer.py +++ b/lib/nlp/lemmatizer.py @@ -3,11 +3,12 @@ # constants from lib.singleton import Singleton +import six WORDNET_LEMMATIZER = 'WORDNET_LEMMATIZER' -class Lemmatizer(object): +class Lemmatizer(six.with_metaclass(Singleton, object)): """ Detects the lemma of a word. Its a wrapper class which can be used to call respective lemmatizer library. Currently, It has been integrated @@ -42,8 +43,6 @@ class Lemmatizer(object): """ - __metaclass__ = Singleton - def __init__(self, lemmatizer_selected=WORDNET_LEMMATIZER): """Initializes a Lemmatizer object diff --git a/lib/nlp/pos.py b/lib/nlp/pos.py index 4c5471a9a..a5d0af188 100644 --- a/lib/nlp/pos.py +++ b/lib/nlp/pos.py @@ -3,6 +3,7 @@ # constants from lib.singleton import Singleton +import six DEFAULT_NLTK_TAGGER_PATH = 'taggers/maxent_treebank_pos_tagger/english.pickle' NLTK_MAXENT_TAGGER = 'NLTK_MAXENT_TAGGER' @@ -19,8 +20,7 @@ def tag(self, tokens, tagset=None): return tagged_tokens -class POS(object): - __metaclass__ = Singleton +class POS(six.with_metaclass(Singleton, object)): def __init__(self, tagger_selected=NLTK_AP_TAGGER, tagger=None): self.tagger_dict = { diff --git a/lib/nlp/stemmer.py b/lib/nlp/stemmer.py index 9adcfd3c7..e98033d18 100644 --- a/lib/nlp/stemmer.py +++ b/lib/nlp/stemmer.py @@ -3,11 +3,12 @@ # constants from lib.singleton import Singleton +import six PORTER_STEMMER = 'PORTER_STEMMER' -class Stemmer(object): +class Stemmer(six.with_metaclass(Singleton, object)): """ Detects the stemmer of a word. Its a wrapper class which can be used to call respective stemmer library. Currently, It has been integrated @@ -36,8 +37,6 @@ class Stemmer(object): """ - __metaclass__ = Singleton - def __init__(self, stemmer_selected=PORTER_STEMMER): """Initializes a Stemmer object diff --git a/lib/nlp/tokenizer.py b/lib/nlp/tokenizer.py index df29d6e5e..bd593ec63 100644 --- a/lib/nlp/tokenizer.py +++ b/lib/nlp/tokenizer.py @@ -5,6 +5,7 @@ # constants from lib.singleton import Singleton +import six NLTK_TOKENIZER = 'WORD_TOKENIZER' PRELOADED_NLTK_TOKENIZER = 'PRELOADED_NLTK_TOKENIZER' @@ -12,7 +13,7 @@ WHITESPACE_TOKENIZER = 'WHITESPACE_TOKENIZER' -class Tokenizer(object): +class Tokenizer(six.with_metaclass(Singleton, object)): """ Returns the list of the tokens from the text Its a wrapper class which can be used to call respective tokenizer library. Currently, It has been integrated @@ -31,8 +32,6 @@ class Tokenizer(object): """ - __metaclass__ = Singleton - def __init__(self, tokenizer_selected=NLTK_TOKENIZER): """Initializes a Tokenizer object diff --git a/models/crf_v2/get_crf_tagger.py b/models/crf_v2/get_crf_tagger.py index dd661adca..31c5eb0d5 100644 --- a/models/crf_v2/get_crf_tagger.py +++ b/models/crf_v2/get_crf_tagger.py @@ -3,13 +3,13 @@ from chatbot_ner.config import CRF_MODEL_S3_BUCKET_REGION, CRF_MODEL_S3_BUCKET_NAME, ner_logger from lib.aws_utils import read_model_dict_from_s3 import pycrfsuite +import six -class CrfModel(object): +class CrfModel(six.with_metaclass(Singleton, object)): """ This class is used to load the crf tagger for a given entity """ - __metaclass__ = Singleton def __init__(self, entity_name): """ diff --git a/models/crf_v2/load_word_embeddings.py b/models/crf_v2/load_word_embeddings.py index 0916af853..bd83b611c 100644 --- a/models/crf_v2/load_word_embeddings.py +++ b/models/crf_v2/load_word_embeddings.py @@ -7,13 +7,13 @@ import json from models.crf_v2.constants import TEXT_LIST, CRF_WORD_EMBEDDINGS_LIST from chatbot_ner.config import ner_logger +import six -class LoadWordEmbeddings(object): +class LoadWordEmbeddings(six.with_metaclass(Singleton, object)): """ This method is used to load the word_embeddings into the memory """ - __metaclass__ = Singleton def __init__(self): """ diff --git a/ner_v1/detectors/base_detector.py b/ner_v1/detectors/base_detector.py index 048955da0..80ca4ef11 100644 --- a/ner_v1/detectors/base_detector.py +++ b/ner_v1/detectors/base_detector.py @@ -21,7 +21,7 @@ _re_flags = re.UNICODE -class BaseDetector(object): +class BaseDetector(six.with_metaclass(abc.ABCMeta, object)): """ This class is the base class from which will be inherited by individual detectors. It primarily contains the priority flow of structured value, message and fallback value and is also use to extend language support for @@ -32,7 +32,6 @@ class BaseDetector(object): _target_language_script (str): ISO-639 language code in which detector would process the query _translation_enabled (bool): Decides to either enable or disable translation API """ - __metaclass__ = abc.ABCMeta def __init__(self, source_language_script=ENGLISH_LANG, translation_enabled=False): """ From f79f50e2acefec6081554790398de6411acbe1f9 Mon Sep 17 00:00:00 2001 From: Raza Sayed Date: Wed, 19 Feb 2020 11:42:38 +0000 Subject: [PATCH 3/4] Using list comprehension instead of filter filter returns a lazy iterable object in python 3. Can use list() to convert the result to a list but i think using list comprehension here is more idiomatic. --- lib/nlp/etc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/nlp/etc.py b/lib/nlp/etc.py index ba8141ac0..2a23385a4 100644 --- a/lib/nlp/etc.py +++ b/lib/nlp/etc.py @@ -37,10 +37,10 @@ def filter_list(list_to_filter, remove_elements): Returns: list of str: The list of elements in first list but not in second - For example output = remove_one_list_to_another(list1=['hi', 'hello', 'bye'],['hi', 'bye']) - print output + For example, output = filter_list(['hi', 'hello', 'bye'],['hi', 'bye']) + print(output) >> ['hello'] """ - return filter(lambda token: token not in remove_elements, list_to_filter) + return [item for item in list_to_filter if item not in remove_elements] From 83eab33335fc019c74a27fc79932c1b7252c0a44 Mon Sep 17 00:00:00 2001 From: Raza Sayed Date: Wed, 19 Feb 2020 11:44:29 +0000 Subject: [PATCH 4/4] Using six.moves.urllib and six.text_type Python 3 has urllib.request and python 2 has urllib. Hence using six.moves.urllib to make the code compatible with both 2 and 3. Also six.text_type evaluates to unicode() for python 2 and str for python 3. --- language_utilities/utils.py | 16 +++++++++++++--- ner_v1/chatbot/entity_detection.py | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/language_utilities/utils.py b/language_utilities/utils.py index dfead17c0..95e7059ce 100644 --- a/language_utilities/utils.py +++ b/language_utilities/utils.py @@ -2,7 +2,9 @@ from language_utilities.constant import ENGLISH_LANG from language_utilities.constant import TRANSLATED_TEXT from chatbot_ner.config import ner_logger, GOOGLE_TRANSLATE_API_KEY -import urllib +import six.moves.urllib.request +import six.moves.urllib.parse +import six.moves.urllib.error import requests @@ -18,8 +20,16 @@ def unicode_urlencode(params): (str): url string with params encoded """ if isinstance(params, dict): - params = params.items() - return urllib.urlencode([(k, isinstance(v, unicode) and v.encode('utf-8') or v) for k, v in params]) + params = list(params.items()) + return six.moves.urllib.parse.urlencode( + [( + k, isinstance( + v, + six.text_type + ) and v.encode('utf-8') or v + ) for (k, v) in params + ] + ) def translate_text(text, source_language_code, target_language_code=ENGLISH_LANG): diff --git a/ner_v1/chatbot/entity_detection.py b/ner_v1/chatbot/entity_detection.py index af57863ff..f1fd5e186 100644 --- a/ner_v1/chatbot/entity_detection.py +++ b/ner_v1/chatbot/entity_detection.py @@ -1077,7 +1077,7 @@ def output_entity_dict_list(entity_value_list, original_text_list, detection_met entity_list = [] for i, entity_value in enumerate(entity_value_list): - if type(entity_value) in [str, unicode]: + if type(entity_value) in [str, six.text_type]: entity_value = { ENTITY_VALUE_DICT_KEY: entity_value }