Merge pull request #540 from hellohaptik/develop

Develop to Master

.gitignore

@@ -107,3 +107,5 @@ logs/*.log*
 newman_reports/
 sftp-config.json
 dev.json
+
+!/dev_docker/*

dev_docker/Dockerfile-python3

@@ -0,0 +1,27 @@
+FROM python:3.6.15
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y wget build-essential curl nginx supervisor && \
+    curl -s https://raw.githubusercontent.com/envkey/envkey-source/master/install.sh | bash
+WORKDIR /app
+COPY requirements.txt nltk_setup.py /app/
+RUN touch /app/config && \
+    pip install --no-cache-dir -U pip &&  \
+    pip install --no-cache-dir -U Cython==0.28.2 newrelic==3.4.0.95 &&  \
+    pip install --no-cache-dir -I uwsgi==2.0.19.1 &&  \
+    pip install --no-cache-dir -r /app/requirements.txt && \
+    pip check && \
+    python /app/nltk_setup.py
+ENV NAME="chatbot_ner"
+ENV DJANGODIR=/app
+ENV NUM_WORKERS=4
+ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings
+ENV PORT=8081
+ENV TIMEOUT=600
+ENV MAX_REQUESTS=1000
+# ENV ENVKEY=
+# ENV NEWRELIC_LICENSE_KEY=
+EXPOSE 8081
+ADD . /app
+RUN chmod +x /app/entrypoint.sh
+CMD /app/entrypoint.sh

dev_docker/docker-compose.yml

@@ -0,0 +1,35 @@
+version: '3.1'
+
+services:
+  chatbot-ner:
+    image: chatbot_ner:latest
+    build:
+      context: .
+      dockerfile: Dockerfile-python3
+    restart: always
+    env_file:
+    - .env
+    volumes:
+    - .:/app/
+    ports:
+    - 8081:8081
+    networks:
+    - ner
+
+
+  neres:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0
+    restart: always
+    environment:
+      - "ES_JAVA_OPTS=-Xmx512m -Xms512m"
+      - "xpack.security.enabled=false"
+    volumes:
+    - ./dbdata/esdata/v1:/usr/share/elasticsearch/data
+    ports:
+    - 9200:9200
+    networks:
+    - ner
+
+networks:
+  ner:
+    driver: bridge

dev_docker/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+export $(envkey-source -f | sed -e 's/\<export\>//g' | sed -e 's/\s\+/\n/g' | xargs) > /dev/null
+export ENVKEY=""
+
+cd $DJANGODIR
+
+export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE
+export PYTHONPATH=$DJANGODIR:$PYTHONPATH
+
+# Create the run directory if it doesn't exist
+# RUNDIR=$(dirname $SOCKFILE)
+# test -d $RUNDIR || mkdir -p $RUNDIR
+
+
+if [ "$NEWRELIC" == "true" ]; then
+
+        # Define app_name based on the environment to be deployed on.
+        if [ "$HAPTIK_ENV" == "staging" ]; then
+                sed -i "s/chatbot_ner/chatbot_ner_staging/g" newrelic.ini
+        fi
+
+        # Update the license key
+        sed -i "s/license_value/$NEWRELIC_LICENSE_KEY/g" newrelic.ini
+
+        # Start uwsgi with newrelic agent
+        export NEW_RELIC_CONFIG_FILE=newrelic.ini
+        newrelic-admin run-program uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000
+
+else
+
+    uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --vacuum --single-interpreter --buffer-size=15000 --http-keepalive=1 --http-timeout=600
+
+fi

docs/install.md

@@ -46,18 +46,46 @@ Following are the steps to create the Docker image and run NER with Docker.
 
      Please follow the installation document: https://docs.docker.com/docker-for-mac/install/
 
-2. **Bring up chatbot_ner:**
-
-```shell
-git clone https://github.com/hellohaptik/chatbot_ner.git
-cd chatbot_ner 
-cp .env.example docker/.env    # (This will have all the basic environment variables to get started, You can update values accordingly)
-cd docker
-docker-compose up --build -d
-```
-
-Open `docker/.env` file and edit the environment variables if needed. (You should change the SECRET_KEY).
-
+2. **Cloning chatbot_ner :**
+  ```bash
+  git clone https://github.com/hellohaptik/chatbot_ner.git
+  cd chatbot_ner
+  # copying dev setup files
+  cp ./dev_docker/* ./
+  ```
+
+3. **Seting up configuration:**
+  Setting up configuration: These setups should be in the parent level directory. Which means inside ~/chatbot_ner/
+  - Copy file `.env.example` to `.env` and modify it based on requirements.
+    - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14). If you don't have access to the notion doc, contact your manager or someone from the ML team.
+  - In `.env` file you should change the `SECRET_KEY`.
+  - Update permission for `entrypoint.sh` file
+  ```bash
+  sudo chmod 777 entrypoint.sh
+  ```
+4. **Bring up chatbot_ner:**
+  - Now its time to build the containers and bring them up.
+  ```bash
+  docker-compose up --build -d
+  ```
+  - Verify if containers are up by docker-compose ps. Keep monitoring for about 1 minute to check if the elasticsearch container is being restarted.
+
+  - In case the elasticsearch container keeps restarting. There are some permission issues with the local folder used by the elasticsearch container. There should be a folder created by elasticsearch with path ~/chatbot_ner/dbdata/esdata/v1.
+
+  - Now bring down the containers and Provide required permission to the above folder. Then bring up the containers.
+  ```bash
+  cd ~/chatbot_ner
+  # stop containers
+  docker-compose down
+  # verify containers are stopped
+  docker-compose ps
+  # give permission to folder
+  sudo chmod -r 777 ./dbdata/esdata/v1
+  # start the containers
+  docker-compose up --build -d
+  ```
+
+The above will also mount the local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner.
 The above will also mount local repo root directory inside the containers /app directory.
 Please wait 5 seconds to run the first curl or do an API call to chatbot_ner.
    > **NOTE**: make sure that nothing is running on port 8081 on your server or your local environment.
@@ -69,7 +97,7 @@ Please wait 5 seconds to run the first curl or do an API call to chatbot_ner.
      Now, on your local machine curl the chatbot api as shown shown below,
      host can be your local machine or a server IP on which you have been running docker on.
 
-   > Port mapping can be changed in docker-compose yml 
+   > Port mapping can be changed in docker-compose yml
 
 **Container commands:**
 
@@ -78,9 +106,9 @@ cd ~/chatbot_ner/docker  # (all compose commands from docker directory of repo)
 docker-compose ps or docker ps # (shows list of running container)
 docker exec -it container-name bash  # (login to container shell)
 docker-compose down # (to kill containers)
-docker-compose restart # (to restart containers, probably when you make code changes) 
+docker-compose restart # (to restart containers, probably when you make code changes)
    ```
-**Check logs** 
+**Check logs**
    ```shell
 docker logs -f docker_chatbot-ner_1
    ```
@@ -99,7 +127,7 @@ tail -f *.log
 
    ```python
 entities = ['date','time','restaurant']
-message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" 
+message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation"
    ```
 
    ```shell

requirements.txt

@@ -17,7 +17,7 @@ pandas==1.0.5
 urllib3==1.26.5
 requests==2.26.0
 requests-aws4auth==0.9
-Django==3.2.13
+Django==3.2.19
 
 elasticsearch==5.5.3
 nltk==3.4.5
@@ -51,7 +51,6 @@ nose-exclude==0.5.0
 # Other convenience libs, ideally should only be installed in dev
 flake8==3.4.1
 jedi==0.17.2
-ipython==7.16.3
 
 #logging
 structlog==21.1.0