From f479b1486b311cf739539539a779da006ca389ec Mon Sep 17 00:00:00 2001 From: harjinder7 Date: Thu, 25 May 2023 08:15:50 +0000 Subject: [PATCH] adding setup files in seperate folder --- .gitignore | 2 ++ dev_docker/Dockerfile-python3 | 27 ++++++++++++++++ dev_docker/docker-compose.yml | 35 ++++++++++++++++++++ dev_docker/entrypoint.sh | 34 ++++++++++++++++++++ docs/install.md | 60 +++++++++++++++++++++++++---------- 5 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 dev_docker/Dockerfile-python3 create mode 100644 dev_docker/docker-compose.yml create mode 100644 dev_docker/entrypoint.sh diff --git a/.gitignore b/.gitignore index 20606789..b49c2ab9 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,5 @@ logs/*.log* newman_reports/ sftp-config.json dev.json + +!/dev_docker/* \ No newline at end of file diff --git a/dev_docker/Dockerfile-python3 b/dev_docker/Dockerfile-python3 new file mode 100644 index 00000000..645081ce --- /dev/null +++ b/dev_docker/Dockerfile-python3 @@ -0,0 +1,27 @@ +FROM python:3.6.15 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y wget build-essential curl nginx supervisor && \ + curl -s https://raw.githubusercontent.com/envkey/envkey-source/master/install.sh | bash +WORKDIR /app +COPY requirements.txt nltk_setup.py /app/ +RUN touch /app/config && \ + pip install --no-cache-dir -U pip && \ + pip install --no-cache-dir -U Cython==0.28.2 newrelic==3.4.0.95 && \ + pip install --no-cache-dir -I uwsgi==2.0.19.1 && \ + pip install --no-cache-dir -r /app/requirements.txt && \ + pip check && \ + python /app/nltk_setup.py +ENV NAME="chatbot_ner" +ENV DJANGODIR=/app +ENV NUM_WORKERS=4 +ENV DJANGO_SETTINGS_MODULE=chatbot_ner.settings +ENV PORT=8081 +ENV TIMEOUT=600 +ENV MAX_REQUESTS=1000 +# ENV ENVKEY= +# ENV NEWRELIC_LICENSE_KEY= +EXPOSE 8081 +ADD . /app +RUN chmod +x /app/entrypoint.sh +CMD /app/entrypoint.sh \ No newline at end of file diff --git a/dev_docker/docker-compose.yml b/dev_docker/docker-compose.yml new file mode 100644 index 00000000..f8878510 --- /dev/null +++ b/dev_docker/docker-compose.yml @@ -0,0 +1,35 @@ +version: '3.1' + +services: + chatbot-ner: + image: chatbot_ner:latest + build: + context: . + dockerfile: Dockerfile-python3 + restart: always + env_file: + - .env + volumes: + - .:/app/ + ports: + - 8081:8081 + networks: + - ner + + + neres: + image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + restart: always + environment: + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + - "xpack.security.enabled=false" + volumes: + - ./dbdata/esdata/v1:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - ner + +networks: + ner: + driver: bridge \ No newline at end of file diff --git a/dev_docker/entrypoint.sh b/dev_docker/entrypoint.sh new file mode 100644 index 00000000..c3a74d33 --- /dev/null +++ b/dev_docker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +export $(envkey-source -f | sed -e 's/\//g' | sed -e 's/\s\+/\n/g' | xargs) > /dev/null +export ENVKEY="" + +cd $DJANGODIR + +export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE +export PYTHONPATH=$DJANGODIR:$PYTHONPATH + +# Create the run directory if it doesn't exist +# RUNDIR=$(dirname $SOCKFILE) +# test -d $RUNDIR || mkdir -p $RUNDIR + + +if [ "$NEWRELIC" == "true" ]; then + + # Define app_name based on the environment to be deployed on. + if [ "$HAPTIK_ENV" == "staging" ]; then + sed -i "s/chatbot_ner/chatbot_ner_staging/g" newrelic.ini + fi + + # Update the license key + sed -i "s/license_value/$NEWRELIC_LICENSE_KEY/g" newrelic.ini + + # Start uwsgi with newrelic agent + export NEW_RELIC_CONFIG_FILE=newrelic.ini + newrelic-admin run-program uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --http-keepalive --vacuum --single-interpreter --buffer-size=15000 + +else + + uwsgi --wsgi-file chatbot_ner/wsgi.py --http :$PORT --strict --need-app --master --workers=$NUM_WORKERS --threads 5 --enable-threads --disable-logging --log-5xx --log-prefix=uwsgi --log-slow=3000 --logto=/app/logs/ner_log.log --logfile-chmod=644 --max-requests=$MAX_REQUESTS --harakiri=$TIMEOUT --reload-mercy=60 --worker-reload-mercy=60 --thunder-lock --http-auto-chunked --vacuum --single-interpreter --buffer-size=15000 --http-keepalive=1 --http-timeout=600 + +fi \ No newline at end of file diff --git a/docs/install.md b/docs/install.md index 04a7dd15..ed7e1c4c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -46,18 +46,46 @@ Following are the steps to create the Docker image and run NER with Docker. Please follow the installation document: https://docs.docker.com/docker-for-mac/install/ -2. **Bring up chatbot_ner:** - -```shell -git clone https://github.com/hellohaptik/chatbot_ner.git -cd chatbot_ner -cp .env.example docker/.env # (This will have all the basic environment variables to get started, You can update values accordingly) -cd docker -docker-compose up --build -d -``` - -Open `docker/.env` file and edit the environment variables if needed. (You should change the SECRET_KEY). - +2. **Cloning chatbot_ner :** + ```bash + git clone https://github.com/hellohaptik/chatbot_ner.git + cd chatbot_ner + # copying dev setup files + cp ./dev_docker/* ./ + ``` + +3. **Seting up configuration:** + Setting up configuration: These setups should be in the parent level directory. Which means inside ~/chatbot_ner/ + - Copy file `.env.example` to `.env` and modify it based on requirements. + - **NOTE** : For updating the .env file Haptik employee can refer to internal notion doc [here](https://www.notion.so/hellohaptik/Alternate-dev-setup-for-NER-env-file-Entity-syncing-7f47ee691aed41c3b7025f2b1976bd14). If you don't have access to the notion doc, contact your manager or someone from the ML team. + - In `.env` file you should change the `SECRET_KEY`. + - Update permission for `entrypoint.sh` file + ```bash + sudo chmod 777 entrypoint.sh + ``` +4. **Bring up chatbot_ner:** + - Now its time to build the containers and bring them up. + ```bash + docker-compose up --build -d + ``` + - Verify if containers are up by docker-compose ps. Keep monitoring for about 1 minute to check if the elasticsearch container is being restarted. + + - In case the elasticsearch container keeps restarting. There are some permission issues with the local folder used by the elasticsearch container. There should be a folder created by elasticsearch with path ~/chatbot_ner/dbdata/esdata/v1. + + - Now bring down the containers and Provide required permission to the above folder. Then bring up the containers. + ```bash + cd ~/chatbot_ner + # stop containers + docker-compose down + # verify containers are stopped + docker-compose ps + # give permission to folder + sudo chmod -r 777 ./dbdata/esdata/v1 + # start the containers + docker-compose up --build -d + ``` + +The above will also mount the local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. The above will also mount local repo root directory inside the containers /app directory. Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. > **NOTE**: make sure that nothing is running on port 8081 on your server or your local environment. @@ -69,7 +97,7 @@ Please wait 5 seconds to run the first curl or do an API call to chatbot_ner. Now, on your local machine curl the chatbot api as shown shown below, host can be your local machine or a server IP on which you have been running docker on. - > Port mapping can be changed in docker-compose yml + > Port mapping can be changed in docker-compose yml **Container commands:** @@ -78,9 +106,9 @@ cd ~/chatbot_ner/docker # (all compose commands from docker directory of repo) docker-compose ps or docker ps # (shows list of running container) docker exec -it container-name bash # (login to container shell) docker-compose down # (to kill containers) -docker-compose restart # (to restart containers, probably when you make code changes) +docker-compose restart # (to restart containers, probably when you make code changes) ``` -**Check logs** +**Check logs** ```shell docker logs -f docker_chatbot-ner_1 ``` @@ -99,7 +127,7 @@ tail -f *.log ```python entities = ['date','time','restaurant'] -message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" +message = "Reserve me a table today at 6:30pm at Mainland China and on Monday at 7:00pm at Barbeque Nation" ``` ```shell