Merge pull request #442 from hellohaptik/develop

Develop to Master 2021-10-13

.env.example

@@ -0,0 +1,62 @@
+# This is .env.example file for chatbot_ner module to hold settings
+# Copy it to a docker/.env and fill in all the values.
+# Never push your personal keys and passwords to any public repository!
+# Please don't add spaces around '='
+
+NAME=chatbot_ner
+DJANGODIR=/app
+ENVIRONMENT=development
+DJANGO_LOG_LEVEL=DEBUG
+DJANGO_SETTINGS_MODULE=chatbot_ner.settings
+DJANGO_WSGI_MODULE=chatbot_ner/wsgi.py
+# Important: Change the value of SECRET_KEY to something else and keep it secret
+SECRET_KEY=!yqqcz-v@(s@kpygpvomcuu3il0q1&qtpz)e_g0ulo-sdv%c0c
+
+NUM_WORKERS=1
+MAX_REQUESTS=1000
+PORT=8081
+TIMEOUT=600
+
+# This is the primary engine to use for datastore. Valid values are one of the following: ['elasticsearch']
+ENGINE=elasticsearch
+
+# ES prefixed variables correspond to settings for elasticsearch.
+# ES_URL is the complete url with auth name and password required to connect. If provided, this will override ES_HOST,
+# ES_PORT, ES_AUTH_NAME, ES_AUTH_PASSWORD
+# ES_HOST and ES_PORT by default is host for ES that comes up with compose
+
+ES_URL=
+ES_AUTH_NAME=
+ES_AUTH_PASSWORD=
+ES_SCHEME=http
+ES_HOST=elasticsearch
+ES_PORT=9200
+ES_ALIAS=entity_data
+ES_INDEX_1=entity_data_v1
+ES_INDEX_2=
+ES_DOC_TYPE=data_dictionary
+ELASTICSEARCH_CRF_DATA_INDEX_NAME=entity_examples_data
+ELASTICSEARCH_CRF_DATA_DOC_TYPE=training_dictionary
+
+ES_REQUEST_TIMEOUT=20
+ES_BULK_MSG_SIZE=1000
+ES_SEARCH_SIZE=10000
+
+# Auth variables if ES is hosted on AWS
+ES_AWS_ACCESS_KEY_ID=
+ES_AWS_REGION=
+ES_AWS_SECRET_ACCESS_KEY=
+ES_AWS_SERVICE=
+
+DESTINATION_ES_SCHEME=
+DESTINATION_HOST=
+DESTINATION_PORT=
+
+# In order to enable entity detection for multiple languages, we use google translate. Please enter the key(optional)
+GOOGLE_TRANSLATE_API_KEY=
+
+# Application performance and error alerting
+ELASTIC_APM_ENABLED=False
+ELASTIC_APM_SERVER_URL=
+SENTRY_ENABLED=False
+SENTRY_DSN=

.gitignore

@@ -3,9 +3,6 @@ __pycache__/
 *.py[cod]
 *$py.class
 
-# haptik ner config file
-config
-
 # C Extensions
 *.so
 
@@ -79,6 +76,7 @@ celerybeat-schedule
 
 # dotenv
 .env
+!.env.example
 update_env.sh
 
 # virtualenv
@@ -94,16 +92,18 @@ ENV/
 .idea/
 .realsync
 
-/Dockerfile*
-/post-merge
-/entrypoint.sh
-/setup.py
-/docker-compose.yml
-/newrelic.ini
-sftp-config.json
+Dockerfile*
+post-merge
+entrypoint.sh
+setup.py
+docker-compose*.yml
+.dockerignore
+newrelic.ini
+dbdata/
 .DS_Store
 logs/*.log*
 
 .vscode
 newman_reports/
+sftp-config.json
 dev.json

chatbot_ner/asgi.py

@@ -0,0 +1,17 @@
+# pylint: disable=W1618
+"""
+ASGI config for chatbot_ner project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'chatbot_ner.settings')
+
+application = get_asgi_application()

chatbot_ner/settings.py

@@ -1,11 +1,11 @@
 """
-Django settings for predictive_server project.
+Django settings for chatbot_ner project.
 
 For more information on this file, see
-https://docs.djangoproject.com/en/1.11/topics/settings/
+https://docs.djangoproject.com/en/3.2/topics/settings/
 
 For the full list of settings and their values, see
-https://docs.djangoproject.com/en/1.11/ref/settings/
+https://docs.djangoproject.com/en/3.2/ref/settings/
 """
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -20,24 +20,17 @@
 ENVIRONMENT = os.environ.get('ENVIRONMENT') or os.environ.get('HAPTIK_ENV')
 
 # Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
-
 TEMPLATE_DEBUG = False
-
 ALLOWED_HOSTS = ['*']
 
-# setup sentry
-
-setup_sentry()
-
 # Application definition
-
 INSTALLED_APPS = [
     'django.contrib.admin',
     'django.contrib.auth',
@@ -48,10 +41,12 @@
     'datastore',
     'ner_v1',
     'ner_v2',
+    # TODO: drop dependency on `nose`, no longer actively maintained
     'django_nose'
 ]
 
 MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
@@ -60,100 +55,132 @@
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
+ROOT_URLCONF = 'chatbot_ner.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'chatbot_ner.wsgi.application'
+
+# Internationalization
+# https://docs.djangoproject.com/en/3.2/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+TIME_ZONE = 'UTC'
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+STATIC_URL = '/static/'
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+# Database
+# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+# setup sentry
+setup_sentry()
+
 # APM
 _elastic_apm_enabled = (os.environ.get('ELASTIC_APM_ENABLED') or '').strip().lower()
 ELASTIC_APM_ENABLED = (_elastic_apm_enabled == 'true') and 'test' not in sys.argv
-ELASTIC_APM_SERVER_URL = os.environ.get('ELASTIC_APM_SERVER_URL')
+
 if ELASTIC_APM_ENABLED:
+    ELASTIC_APM_SERVER_URL = os.environ.get('ELASTIC_APM_SERVER_URL')
     ELASTIC_APM = {
         'DEBUG': DEBUG,
         'SERVICE_NAME': 'chatbot_ner',
         'SERVER_URL': ELASTIC_APM_SERVER_URL,
         'SPAN_FRAMES_MIN_DURATION': '5ms',
         'STACK_TRACE_LIMIT': 500,
         'ENVIRONMENT': ENVIRONMENT,
-        'TRANSACTION_SAMPLE_RATE': '0.1',
+        'TRANSACTION_SAMPLE_RATE': 0.1,
         'TRANSACTION_MAX_SPANS': 500,
-        'INSTRUMENT': 'True',
-        'DISABLE_SEND': 'False',
+        'INSTRUMENT': True,
+        'DISABLE_SEND': False,
         'CAPTURE_BODY': 'off',
         'SERVER_TIMEOUT': '2s',
+        'API_REQUEST_TIME': '10s',
+        'DJANGO_AUTOINSERT_MIDDLEWARE': False,
+        'DISABLE_LOG_RECORD_FACTORY': True,
     }
     INSTALLED_APPS.append('elasticapm.contrib.django')
     MIDDLEWARE.append('elasticapm.contrib.django.middleware.TracingMiddleware')
 
-ROOT_URLCONF = 'chatbot_ner.urls'
-
-WSGI_APPLICATION = 'chatbot_ner.wsgi.application'
-
-
-# Database
-# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
-
-# FOR TEST CASES - COMMON SETTINGS FOR ALL ENVIRONMENTS
-
-
-class DisableMigrations(object):
-
-    def __contains__(self, item):
-        return True
-
-    def __getitem__(self, item):
-        return None
-
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    }
-}
-
-# Keeping this block here for ease in the future
-TEST_DB_PATH = os.environ.get('TEST_DB_PATH') or '/dev/shm/chatbot_ner_test.db.sqlite3'
+TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 if 'test' in sys.argv:
+    # FOR TEST CASES - COMMON SETTINGS FOR ALL ENVIRONMENTS
+    TEST_DB_PATH = os.environ.get('TEST_DB_PATH') or '/dev/shm/chatbot_ner_test.db.sqlite3'
     DATABASES['default'] = {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': TEST_DB_PATH,
         'CONN_MAX_AGE': 60
     }
-
-TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
-NOSE_ARGS = [
-    '--nocapture',
-    '--nologcapture',
-    '--verbosity=3',
-    '--exclude-dir=chatbot_ner/',
-    '--exclude-dir=docs/',
-    '--exclude-dir=docker/',
-    '--exclude-dir=data/',
-    '--ignore-files=manage.py',
-    '--ignore-files=nltk_setup.py',
-    '--ignore-files=__init__.py',
-    '--ignore-files=const.py',
-    '--ignore-files=constant.py',
-    '--ignore-files=constants.py',
-    '--ignore-files=run_postman_tests.py',
-    '--cover-erase',
-    '--cover-package=datastore,external_api,language_utilities,lib,ner_v1,ner_v2',
-    '--cover-inclusive',
-]
-
-# Internationalization
-# https://docs.djangoproject.com/en/1.11/topics/i18n/
-
-LANGUAGE_CODE = 'en-us'
-
-TIME_ZONE = 'UTC'
-
-USE_I18N = True
-
-USE_L10N = True
-
-USE_TZ = True
-
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.11/howto/static-files/
-
-STATIC_URL = '/static/'
+    MIGRATION_MODULES = {
+        'datastore': None,
+        'ner_v1': None,
+        'ner_v2': None,
+    }
+    NOSE_ARGS = [
+        '--nocapture',
+        '--nologcapture',
+        '--verbosity=3',
+        '--exclude-dir=chatbot_ner/',
+        '--exclude-dir=docs/',
+        '--exclude-dir=docker/',
+        '--exclude-dir=data/',
+        '--ignore-files=manage.py',
+        '--ignore-files=nltk_setup.py',
+        '--ignore-files=__init__.py',
+        '--ignore-files=const.py',
+        '--ignore-files=constant.py',
+        '--ignore-files=constants.py',
+        '--ignore-files=run_postman_tests.py',
+        '--cover-erase',
+        '--cover-package=datastore,external_api,language_utilities,lib,ner_v1,ner_v2',
+        '--cover-inclusive',
+    ]