CHATBOT_TFIDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# JARVIS LAWYER  APPRENTICE BOT- TFIDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Required Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "import nltk\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize \n",
    "from nltk import pos_tag\n",
    "import numpy as np\n",
    "import pickle\n",
    "import string\n",
    "import random\n",
    "import timeit\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer \n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/machine1/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Preparation - Convert CSV to JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MESSAGE</th>\n",
       "      <th>RESPONSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What are the requirements for me to adopt my n...</td>\n",
       "      <td>You may wish to visit this site: https://app.a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>how to have a properly adopted in accordance w...</td>\n",
       "      <td>You may want to refer to this site:https://app...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>we have been raising our grandchild from birth...</td>\n",
       "      <td>You would have to apply to the Family Courts: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I was legally adopted since young. Many years ...</td>\n",
       "      <td>Your biological parents will have to apply to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Can an adoption be reversed? Both adoptive par...</td>\n",
       "      <td>We are not aware of any such cases as the Adop...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             MESSAGE  \\\n",
       "0  What are the requirements for me to adopt my n...   \n",
       "1  how to have a properly adopted in accordance w...   \n",
       "2  we have been raising our grandchild from birth...   \n",
       "3  I was legally adopted since young. Many years ...   \n",
       "4  Can an adoption be reversed? Both adoptive par...   \n",
       "\n",
       "                                            RESPONSE  \n",
       "0  You may wish to visit this site: https://app.a...  \n",
       "1  You may want to refer to this site:https://app...  \n",
       "2  You would have to apply to the Family Courts: ...  \n",
       "3  Your biological parents will have to apply to ...  \n",
       "4  We are not aware of any such cases as the Adop...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/home/machine1/SANDY/text_mining/CHATBOT/1_TFIDF/'\n",
    "convdata = pd.read_csv(path+'legal_help_clean.csv')\n",
    "\n",
    "#show header of the dataset\n",
    "convdata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'MESSAGE': \"What are the requirements for me to adopt my nephew whom I have been caring like my own for 7 years after parents' divorce?\",\n",
       "  'RESPONSE': 'You may wish to visit this site: https://app.adoption.gov.sg/AdoptionProcess.aspx'},\n",
       " {'MESSAGE': 'how to have a properly adopted in accordance with the laws of Singapore for my godson,who is a PR now',\n",
       "  'RESPONSE': 'You may want to refer to this site:https://app.msf.gov.sg/Adoption/How-to-adopt-a-citizen-or-PR'}]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#covert dataframes to json\n",
    "convdata_json = json.loads(convdata.to_json(orient='records'))\n",
    "convdata_json[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export as data as JSON\n",
    "with open(path+'conversation_json.json', 'w') as outfile:\n",
    "    json.dump(convdata_json, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function Defination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#greeting function\n",
    "GREETING_INPUTS = (\"hello\", \"hi\", \"greetings\", \"hello i need help\", \"good day\",\"hey\",\"i need help\", \"greetings\")\n",
    "GREETING_RESPONSES = [\"Good day, How may i of help?\", \"Hello, How can i help?\", \"hello\", \"I am glad! You are talking to me.\"]\n",
    "           \n",
    "def greeting(sentence):\n",
    "    for word in sentence.split():\n",
    "        if word.lower() in GREETING_INPUTS:\n",
    "            return random.choice(GREETING_RESPONSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Wordnet Lemmatization \n",
    "\n",
    "lemmer = nltk.WordNetLemmatizer()\n",
    "def LemTokens(tokens):\n",
    "    return [lemmer.lemmatize(token) for token in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove punctuation\n",
    "def RemovePunction(tokens):\n",
    "    return[t for t in tokens if t not in string.punctuation]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179\n"
     ]
    }
   ],
   "source": [
    "# Create a stopword list from the standard list of stopwords available in nltk\n",
    "stop_words = set(stopwords.words('english'))\n",
    "print(len(stop_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Talk_To_Javris(test_set_sentence):\n",
    "    json_file_path = path+\"conversation_json.json\" \n",
    "    tfidf_vectorizer_pickle_path = path + \"tfidf_vectorizer.pkl\"\n",
    "    tfidf_matrix_pickle_path = path+ \"tfidf_matrix_train.pkl\"\n",
    "    \n",
    "    i = 0\n",
    "    sentences = []\n",
    "    \n",
    "    # ---------------Tokenisation of user input -----------------------------#\n",
    "    \n",
    "    tokens = RemovePunction(nltk.word_tokenize(test_set_sentence))\n",
    "    pos_tokens = [word for word,pos in pos_tag(tokens, tagset='universal')]\n",
    "    \n",
    "    word_tokens = LemTokens(pos_tokens)\n",
    "    \n",
    "    filtered_sentence = []\n",
    "    for w in word_tokens: \n",
    "        if w not in stop_words: \n",
    "            filtered_sentence.append(w)  \n",
    "    \n",
    "    filtered_sentence =\" \".join(filtered_sentence).lower()\n",
    "            \n",
    "    test_set = (filtered_sentence, \"\")\n",
    "    \n",
    "    #For Tracing, comment to remove from print.\n",
    "    #print('USER INPUT:'+filtered_sentence)\n",
    "    \n",
    "    # -----------------------------------------------------------------------#\n",
    "        \n",
    "    try: \n",
    "        # ---------------Use Pre-Train Model------------------#\n",
    "        f = open(tfidf_vectorizer_pickle_path, 'rb')\n",
    "        tfidf_vectorizer = pickle.load(f)\n",
    "        f.close()\n",
    "        \n",
    "        f = open(tfidf_matrix_pickle_path, 'rb')\n",
    "        tfidf_matrix_train = pickle.load(f)\n",
    "        # ---------------------------------------------------#\n",
    "    except: \n",
    "        # ---------------To Train------------------#\n",
    "        \n",
    "        start = timeit.default_timer()\n",
    "        \n",
    "        with open(json_file_path) as sentences_file:\n",
    "            reader = json.load(sentences_file)\n",
    "            \n",
    "            # ---------------Tokenisation of training input -----------------------------#    \n",
    "            \n",
    "            for row in reader:\n",
    "                db_tokens = RemovePunction(nltk.word_tokenize(row['MESSAGE']))\n",
    "                pos_db_tokens = [word for word,pos in pos_tag(db_tokens, tagset='universal')]\n",
    "                db_word_tokens = LemTokens(pos_db_tokens)\n",
    "                \n",
    "                db_filtered_sentence = [] \n",
    "                for dbw in db_word_tokens: \n",
    "                    if dbw not in stop_words: \n",
    "                        db_filtered_sentence.append(dbw)  \n",
    "                \n",
    "                db_filtered_sentence =\" \".join(db_filtered_sentence).lower()\n",
    "                \n",
    "                #Debugging Checkpoint\n",
    "                print('TRAINING INPUT: '+db_filtered_sentence)\n",
    "                \n",
    "                sentences.append(db_filtered_sentence)\n",
    "                i +=1                \n",
    "            # ---------------------------------------------------------------------------#\n",
    "                \n",
    "        tfidf_vectorizer = TfidfVectorizer() \n",
    "        tfidf_matrix_train = tfidf_vectorizer.fit_transform(sentences)\n",
    "        \n",
    "        #train timing\n",
    "        stop = timeit.default_timer()\n",
    "        print (\"Training Time : \")\n",
    "        print (stop - start) \n",
    "    \n",
    "        f = open(tfidf_vectorizer_pickle_path, 'wb')\n",
    "        pickle.dump(tfidf_vectorizer, f) \n",
    "        f.close()\n",
    "    \n",
    "        f = open(tfidf_matrix_pickle_path, 'wb')\n",
    "        pickle.dump(tfidf_matrix_train, f) \n",
    "        f.close \n",
    "        # ------------------------------------------#\n",
    "        \n",
    "    #use the learnt dimension space to run TF-IDF on the query\n",
    "    tfidf_matrix_test = tfidf_vectorizer.transform(test_set)\n",
    "\n",
    "    #then run cosine similarity between the 2 tf-idfs\n",
    "    cosine = cosine_similarity(tfidf_matrix_test, tfidf_matrix_train)\n",
    "    \n",
    "    #if not in the topic trained.no similarity \n",
    "    idx= cosine.argsort()[0][-2]\n",
    "    flat =  cosine.flatten()\n",
    "    flat.sort()\n",
    "    req_tfidf = flat[-2]\n",
    "    \n",
    "    if (req_tfidf==0): #Threshold A\n",
    "        \n",
    "        not_understood = \"Apology, I do not understand. Can you rephrase?\"\n",
    "        \n",
    "        return not_understood, not_understood, 2\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        cosine = np.delete(cosine, 0)\n",
    "\n",
    "        #get the max score\n",
    "        max = cosine.max()\n",
    "        response_index = 0\n",
    "\n",
    "        #if max score is lower than < 0.34 > (we see can ask if need to rephrase.)\n",
    "        if (max <= 0.34): #Threshold B\n",
    "            \n",
    "            not_understood = \"Apology, I do not understand. Can you rephrase?\"\n",
    "            \n",
    "            return not_understood,not_understood, 2\n",
    "        else:\n",
    "\n",
    "                #if score is more than 0.91 list the multi response and get a random reply\n",
    "                if (max > 0.91): #Threshold C\n",
    "                    \n",
    "                    new_max = max - 0.05 \n",
    "                    # load them to a list\n",
    "                    list = np.where(cosine > new_max) \n",
    "                   \n",
    "                    # choose a random one to return to the user \n",
    "                    response_index = random.choice(list[0])\n",
    "                else:\n",
    "                    # else we would simply return the highest score\n",
    "                    response_index = np.where(cosine == max)[0][0] + 2 \n",
    "\n",
    "                j = 0 \n",
    "\n",
    "                with open(json_file_path, \"r\") as sentences_file:\n",
    "                    reader = json.load(sentences_file)\n",
    "                    for row in reader:\n",
    "                        j += 1 \n",
    "                        if j == response_index: \n",
    "                            return row[\"RESPONSE\"], row[\"MESSAGE\"], max\n",
    "                            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "......................................................................................\n",
      "\u001b[1;37;40mJarvis\u001b[0m: My name is Jarvis, a Lawyer Apprentice Bot.\n",
      "\u001b[1;37;40mJarvis\u001b[0m: I will try my best to answer your query.\n",
      "\u001b[1;37;40mJarvis\u001b[0m: If you want to exit, you can type < bye >.\n",
      "......................................................................................\n",
      "USER  :Hello\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: I am glad! You are talking to me.\n",
      "......................................................................................\n",
      "USER  :Can i get divorce?\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: It depends on the specific facts of your case. It may be possible but it will not be straightforward. We suggest you seek legal assistance.\n",
      "......................................................................................\n",
      "USER  :cost of divorce?\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: This depends on the lawyer that you approach.\n",
      "......................................................................................\n",
      "USER  :i want to adopt\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: In order to adopt a child in Singapore, you must fulfil certain guidelines laid down by the Ministry of Social and Family Development. These can be found here: http://app.customerfeedback.mcys.gov.sg/fcd_faqmain.asp?strFAQSysid=2004919155835&strItemChoice=200499134018&strSubItemChoice=2004919114225&action=SHOWTOPICS&m_strTopicSysID=2004919114242 Essentially, the requirements are that you must be a resident of Singapore, be at least 25 years of age and be at least 21 years older than the child. There are other requirements which may found at the link above.\n",
      "......................................................................................\n",
      "USER  :what is the process for adoption\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: Adoption is a legal process where parental rights over a child are conferred to the applicants, who may not be the natural parents of the child. The Court will appoint a Guardian in Adoption (GIA) for the child, whose role is to safeguard the child? s legal interest before the Court. The GIA will investigate the circumstances of the adoption and report to the Court. The Court would only grant an adoption order after it is satisfied that the requirements of the Adoption of Children Act (Cap 4) have been complied with and that it is in the best interests of the child to be adopted.\n",
      "......................................................................................\n",
      "USER  :i want a divorce, do i need a lawyer?\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: No, you can apply for a divorce yourself, which is known in legal terms as  ?acting in person? . However, you will still need to comply with the legal and procedural requirements of the Family Court proceedings. For example, you must ensure that the necessary documents are prepared in the correct form and file these through the Electronic Filing System.The Family Court cannot give you any advice on what you should say or do. Only a qualified lawyer can give you independent legal advice on the merits of your case.\n",
      "......................................................................................\n",
      "USER  :Do i need to pay maintenance? \n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: Maintenance is financial support by your former spouse.\n",
      "......................................................................................\n",
      "USER  :Do i need to sell my matrimonial asset after divorce?\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: If the asset was acquired after marriage, it is a matrimonial asset if (i) it has been used by your spouse, or the children, while you were living together; (ii) it has been substantially improved during the marriage by your spouse, of by both of you.If the asset was a gift, or was inherited by you, it is not a matrimonial asset except if (i) it is the matrimonial home; (ii) it has been substantially improved during the marriage by your spouse, of by both of you.\n",
      "......................................................................................\n",
      "USER  :what is the criteria for alimony\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: Please refer to this page: https://www.familyjusticecourts.gov.sg/Common/Pages/Maintenance.aspx\n",
      "......................................................................................\n",
      "USER  :How the court decide the custody of the child after divorce\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: The Court will generally grant an access order to ensure that the parent without care and control will get to see his or her children regularly. If both parents can cooperate with each other, the Court will usually order  ?reasonable access? and the arrangements can be flexible. If not, the Court may set a timetable for access or impose specific conditions to be complied with.\n",
      "......................................................................................\n",
      "USER  :i want a baby\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: Apology, I do not understand. Can you rephrase?\n",
      "......................................................................................\n",
      "USER  :bye\n",
      "......................................................................................\n",
      "\u001b[1;37;40mJARVIS\u001b[0m: Bye! Hope that i am of help.\n"
     ]
    }
   ],
   "source": [
    "flag=True\n",
    "print(\"......................................................................................\")\n",
    "print('\\x1b[1;37;40m' + 'Jarvis'+'\\x1b[0m'+': '+ 'My name is Jarvis, a Lawyer Apprentice Bot.')\n",
    "print('\\x1b[1;37;40m' + 'Jarvis'+'\\x1b[0m'+': '+ 'I will try my best to answer your query.')\n",
    "print('\\x1b[1;37;40m' + 'Jarvis'+'\\x1b[0m'+': '+ 'If you want to exit, you can type < bye >.')\n",
    "while(flag==True):\n",
    "    print(\"......................................................................................\")\n",
    "    sentence = input('\\x1b[0;30;47m' +\"USER  \"+'\\x1b[0m'+\":\")\n",
    "    print(\"......................................................................................\")\n",
    "    if(sentence.lower()!='bye'):\n",
    "        if(greeting(sentence.lower())!=None):\n",
    "            print('\\x1b[1;37;40m' + 'JARVIS'+'\\x1b[0m'+': '+ greeting(sentence.lower()))\n",
    "        else:\n",
    "            response_primary, response_message, line_id_primary = Talk_To_Javris(sentence)\n",
    "            print('\\x1b[1;37;40m' + 'JARVIS'+'\\x1b[0m'+': '+response_primary)\n",
    "            \n",
    "            #For Tracing, comment to remove from print \n",
    "            #print(\"\")\n",
    "            #print(\"SCORE: \"+str(line_id_primary))\n",
    "            #print(\"COR_QUES:\"+response_message)\n",
    "            #print(\"\")\n",
    "    else:\n",
    "        flag=False\n",
    "print('\\x1b[1;37;40m' + 'JARVIS'+'\\x1b[0m'+': '+\"Bye! Hope that i am of help.\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}