NLP_AMIR_DODANGEH_HW01.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"NLP_AMIR_DODANGEH_HW01.ipynb","provenance":[],"authorship_tag":"ABX9TyPbUgY1uTpodZF0kvx2jVwq"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"16IesyNTX_Bq","executionInfo":{"status":"ok","timestamp":1621598584878,"user_tz":-270,"elapsed":59937,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"8dda4b8c-7074-4e89-d925-93ab87c15f86"},"source":["!pip install textblob \n"," #!pip install --upgrade nltk\n","#!pip install lm\n","#python  -m textblob.download_corpora\n","import nltk\n","nltk.download()"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: textblob in /usr/local/lib/python3.7/dist-packages (0.15.3)\n","Requirement already satisfied: nltk>=3.1 in /usr/local/lib/python3.7/dist-packages (from textblob) (3.2.5)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk>=3.1->textblob) (1.15.0)\n","NLTK Downloader\n","---------------------------------------------------------------------------\n","    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n","---------------------------------------------------------------------------\n","Downloader> d\n","\n","Download which package (l=list; x=cancel)?\n","  Identifier> all\n","    Downloading collection 'all'\n","       | \n","       | Downloading package abc to /root/nltk_data...\n","       |   Unzipping corpora/abc.zip.\n","       | Downloading package alpino to /root/nltk_data...\n","       |   Unzipping corpora/alpino.zip.\n","       | Downloading package biocreative_ppi to /root/nltk_data...\n","       |   Unzipping corpora/biocreative_ppi.zip.\n","       | Downloading package brown to /root/nltk_data...\n","       |   Unzipping corpora/brown.zip.\n","       | Downloading package brown_tei to /root/nltk_data...\n","       |   Unzipping corpora/brown_tei.zip.\n","       | Downloading package cess_cat to /root/nltk_data...\n","       |   Unzipping corpora/cess_cat.zip.\n","       | Downloading package cess_esp to /root/nltk_data...\n","       |   Unzipping corpora/cess_esp.zip.\n","       | Downloading package chat80 to /root/nltk_data...\n","       |   Unzipping corpora/chat80.zip.\n","       | Downloading package city_database to /root/nltk_data...\n","       |   Unzipping corpora/city_database.zip.\n","       | Downloading package cmudict to /root/nltk_data...\n","       |   Unzipping corpora/cmudict.zip.\n","       | Downloading package comparative_sentences to\n","       |     /root/nltk_data...\n","       |   Unzipping corpora/comparative_sentences.zip.\n","       | Downloading package comtrans to /root/nltk_data...\n","       | Downloading package conll2000 to /root/nltk_data...\n","       |   Unzipping corpora/conll2000.zip.\n","       | Downloading package conll2002 to /root/nltk_data...\n","       |   Unzipping corpora/conll2002.zip.\n","       | Downloading package conll2007 to /root/nltk_data...\n","       | Downloading package crubadan to /root/nltk_data...\n","       |   Unzipping corpora/crubadan.zip.\n","       | Downloading package dependency_treebank to /root/nltk_data...\n","       |   Unzipping corpora/dependency_treebank.zip.\n","       | Downloading package dolch to /root/nltk_data...\n","       |   Unzipping corpora/dolch.zip.\n","       | Downloading package europarl_raw to /root/nltk_data...\n","       |   Unzipping corpora/europarl_raw.zip.\n","       | Downloading package floresta to /root/nltk_data...\n","       |   Unzipping corpora/floresta.zip.\n","       | Downloading package framenet_v15 to /root/nltk_data...\n","       |   Unzipping corpora/framenet_v15.zip.\n","       | Downloading package framenet_v17 to /root/nltk_data...\n","       |   Unzipping corpora/framenet_v17.zip.\n","       | Downloading package gazetteers to /root/nltk_data...\n","       |   Unzipping corpora/gazetteers.zip.\n","       | Downloading package genesis to /root/nltk_data...\n","       |   Unzipping corpora/genesis.zip.\n","       | Downloading package gutenberg to /root/nltk_data...\n","       |   Unzipping corpora/gutenberg.zip.\n","       | Downloading package ieer to /root/nltk_data...\n","       |   Unzipping corpora/ieer.zip.\n","       | Downloading package inaugural to /root/nltk_data...\n","       |   Unzipping corpora/inaugural.zip.\n","       | Downloading package indian to /root/nltk_data...\n","       |   Unzipping corpora/indian.zip.\n","       | Downloading package jeita to /root/nltk_data...\n","       | Downloading package kimmo to /root/nltk_data...\n","       |   Unzipping corpora/kimmo.zip.\n","       | Downloading package knbc to /root/nltk_data...\n","       | Downloading package lin_thesaurus to /root/nltk_data...\n","       |   Unzipping corpora/lin_thesaurus.zip.\n","       | Downloading package mac_morpho to /root/nltk_data...\n","       |   Unzipping corpora/mac_morpho.zip.\n","       | Downloading package machado to /root/nltk_data...\n","       | Downloading package masc_tagged to /root/nltk_data...\n","       | Downloading package moses_sample to /root/nltk_data...\n","       |   Unzipping models/moses_sample.zip.\n","       | Downloading package movie_reviews to /root/nltk_data...\n","       |   Unzipping corpora/movie_reviews.zip.\n","       | Downloading package names to /root/nltk_data...\n","       |   Unzipping corpora/names.zip.\n","       | Downloading package nombank.1.0 to /root/nltk_data...\n","       | Downloading package nps_chat to /root/nltk_data...\n","       |   Unzipping corpora/nps_chat.zip.\n","       | Downloading package omw to /root/nltk_data...\n","       |   Unzipping corpora/omw.zip.\n","       | Downloading package opinion_lexicon to /root/nltk_data...\n","       |   Unzipping corpora/opinion_lexicon.zip.\n","       | Downloading package paradigms to /root/nltk_data...\n","       |   Unzipping corpora/paradigms.zip.\n","       | Downloading package pil to /root/nltk_data...\n","       |   Unzipping corpora/pil.zip.\n","       | Downloading package pl196x to /root/nltk_data...\n","       |   Unzipping corpora/pl196x.zip.\n","       | Downloading package ppattach to /root/nltk_data...\n","       |   Unzipping corpora/ppattach.zip.\n","       | Downloading package problem_reports to /root/nltk_data...\n","       |   Unzipping corpora/problem_reports.zip.\n","       | Downloading package propbank to /root/nltk_data...\n","       | Downloading package ptb to /root/nltk_data...\n","       |   Unzipping corpora/ptb.zip.\n","       | Downloading package product_reviews_1 to /root/nltk_data...\n","       |   Unzipping corpora/product_reviews_1.zip.\n","       | Downloading package product_reviews_2 to /root/nltk_data...\n","       |   Unzipping corpora/product_reviews_2.zip.\n","       | Downloading package pros_cons to /root/nltk_data...\n","       |   Unzipping corpora/pros_cons.zip.\n","       | Downloading package qc to /root/nltk_data...\n","       |   Unzipping corpora/qc.zip.\n","       | Downloading package reuters to /root/nltk_data...\n","       | Downloading package rte to /root/nltk_data...\n","       |   Unzipping corpora/rte.zip.\n","       | Downloading package semcor to /root/nltk_data...\n","       | Downloading package senseval to /root/nltk_data...\n","       |   Unzipping corpora/senseval.zip.\n","       | Downloading package sentiwordnet to /root/nltk_data...\n","       |   Unzipping corpora/sentiwordnet.zip.\n","       | Downloading package sentence_polarity to /root/nltk_data...\n","       |   Unzipping corpora/sentence_polarity.zip.\n","       | Downloading package shakespeare to /root/nltk_data...\n","       |   Unzipping corpora/shakespeare.zip.\n","       | Downloading package sinica_treebank to /root/nltk_data...\n","       |   Unzipping corpora/sinica_treebank.zip.\n","       | Downloading package smultron to /root/nltk_data...\n","       |   Unzipping corpora/smultron.zip.\n","       | Downloading package state_union to /root/nltk_data...\n","       |   Unzipping corpora/state_union.zip.\n","       | Downloading package stopwords to /root/nltk_data...\n","       |   Unzipping corpora/stopwords.zip.\n","       | Downloading package subjectivity to /root/nltk_data...\n","       |   Unzipping corpora/subjectivity.zip.\n","       | Downloading package swadesh to /root/nltk_data...\n","       |   Unzipping corpora/swadesh.zip.\n","       | Downloading package switchboard to /root/nltk_data...\n","       |   Unzipping corpora/switchboard.zip.\n","       | Downloading package timit to /root/nltk_data...\n","       |   Unzipping corpora/timit.zip.\n","       | Downloading package toolbox to /root/nltk_data...\n","       |   Unzipping corpora/toolbox.zip.\n","       | Downloading package treebank to /root/nltk_data...\n","       |   Unzipping corpora/treebank.zip.\n","       | Downloading package twitter_samples to /root/nltk_data...\n","       |   Unzipping corpora/twitter_samples.zip.\n","       | Downloading package udhr to /root/nltk_data...\n","       |   Unzipping corpora/udhr.zip.\n","       | Downloading package udhr2 to /root/nltk_data...\n","       |   Unzipping corpora/udhr2.zip.\n","       | Downloading package unicode_samples to /root/nltk_data...\n","       |   Unzipping corpora/unicode_samples.zip.\n","       | Downloading package universal_treebanks_v20 to\n","       |     /root/nltk_data...\n","       | Downloading package verbnet to /root/nltk_data...\n","       |   Unzipping corpora/verbnet.zip.\n","       | Downloading package verbnet3 to /root/nltk_data...\n","       |   Unzipping corpora/verbnet3.zip.\n","       | Downloading package webtext to /root/nltk_data...\n","       |   Unzipping corpora/webtext.zip.\n","       | Downloading package wordnet to /root/nltk_data...\n","       |   Unzipping corpora/wordnet.zip.\n","       | Downloading package wordnet_ic to /root/nltk_data...\n","       |   Unzipping corpora/wordnet_ic.zip.\n","       | Downloading package words to /root/nltk_data...\n","       |   Unzipping corpora/words.zip.\n","       | Downloading package ycoe to /root/nltk_data...\n","       |   Unzipping corpora/ycoe.zip.\n","       | Downloading package rslp to /root/nltk_data...\n","       |   Unzipping stemmers/rslp.zip.\n","       | Downloading package maxent_treebank_pos_tagger to\n","       |     /root/nltk_data...\n","       |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n","       | Downloading package universal_tagset to /root/nltk_data...\n","       |   Unzipping taggers/universal_tagset.zip.\n","       | Downloading package maxent_ne_chunker to /root/nltk_data...\n","       |   Unzipping chunkers/maxent_ne_chunker.zip.\n","       | Downloading package punkt to /root/nltk_data...\n","       |   Unzipping tokenizers/punkt.zip.\n","       | Downloading package book_grammars to /root/nltk_data...\n","       |   Unzipping grammars/book_grammars.zip.\n","       | Downloading package sample_grammars to /root/nltk_data...\n","       |   Unzipping grammars/sample_grammars.zip.\n","       | Downloading package spanish_grammars to /root/nltk_data...\n","       |   Unzipping grammars/spanish_grammars.zip.\n","       | Downloading package basque_grammars to /root/nltk_data...\n","       |   Unzipping grammars/basque_grammars.zip.\n","       | Downloading package large_grammars to /root/nltk_data...\n","       |   Unzipping grammars/large_grammars.zip.\n","       | Downloading package tagsets to /root/nltk_data...\n","       |   Unzipping help/tagsets.zip.\n","       | Downloading package snowball_data to /root/nltk_data...\n","       | Downloading package bllip_wsj_no_aux to /root/nltk_data...\n","       |   Unzipping models/bllip_wsj_no_aux.zip.\n","       | Downloading package word2vec_sample to /root/nltk_data...\n","       |   Unzipping models/word2vec_sample.zip.\n","       | Downloading package panlex_swadesh to /root/nltk_data...\n","       | Downloading package mte_teip5 to /root/nltk_data...\n","       |   Unzipping corpora/mte_teip5.zip.\n","       | Downloading package averaged_perceptron_tagger to\n","       |     /root/nltk_data...\n","       |   Unzipping taggers/averaged_perceptron_tagger.zip.\n","       | Downloading package averaged_perceptron_tagger_ru to\n","       |     /root/nltk_data...\n","       |   Unzipping taggers/averaged_perceptron_tagger_ru.zip.\n","       | Downloading package perluniprops to /root/nltk_data...\n","       |   Unzipping misc/perluniprops.zip.\n","       | Downloading package nonbreaking_prefixes to\n","       |     /root/nltk_data...\n","       |   Unzipping corpora/nonbreaking_prefixes.zip.\n","       | Downloading package vader_lexicon to /root/nltk_data...\n","       | Downloading package porter_test to /root/nltk_data...\n","       |   Unzipping stemmers/porter_test.zip.\n","       | Downloading package wmt15_eval to /root/nltk_data...\n","       |   Unzipping models/wmt15_eval.zip.\n","       | Downloading package mwa_ppdb to /root/nltk_data...\n","       |   Unzipping misc/mwa_ppdb.zip.\n","       | \n","     Done downloading collection all\n","\n","---------------------------------------------------------------------------\n","    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n","---------------------------------------------------------------------------\n","Downloader> q\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"WCbfthDOYaH7","executionInfo":{"status":"ok","timestamp":1621598584880,"user_tz":-270,"elapsed":13,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}}},"source":["# import libraries needed, read the dataset\n","import nltk, re, pprint, string\n","from nltk import word_tokenize, sent_tokenize\n","string.punctuation = string.punctuation +'“'+'”'+'-'+'’'+'‘'+'—'\n","string.punctuation = string.punctuation.replace('.', '')\n"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"grO5HIOlYrmd","executionInfo":{"status":"ok","timestamp":1621598610770,"user_tz":-270,"elapsed":25900,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"a24e167a-6d0d-4417-eac1-1f07fb5fe40d"},"source":["import sys\n","from google.colab import drive\n","drive.mount('/content/gdrive')\n","\n","# Opening and reading the `Input.txt` file\n","file = open(\"/content/gdrive/My Drive/Colab Notebooks/NLP/HW01/corpes.txt\").read()"],"execution_count":4,"outputs":[{"output_type":"stream","text":["Mounted at /content/gdrive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"jjN9QCINY7ae","executionInfo":{"status":"ok","timestamp":1621598610771,"user_tz":-270,"elapsed":28,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}}},"source":["#preprocess data\n","file_nl_removed = \"\"\n","for line in file:\n","  line_nl_removed = line.replace(\"\\n\", \" \")      #removes newlines\n","  file_nl_removed += line_nl_removed\n","file_p = \"\".join([char for char in file_nl_removed if char not in string.punctuation])   #removes all special characters"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nAIyEVK-Y-In","executionInfo":{"status":"ok","timestamp":1621598610773,"user_tz":-270,"elapsed":27,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"1bf0b6a1-192c-4813-941e-e272a6a02161"},"source":["sents = nltk.sent_tokenize(file_p)\n","print(\"The number of sentences is\", len(sents)) \n","#prints the number of sentences\n","words = nltk.word_tokenize(file_p)\n","print(\"The number of tokens is\", len(words)) \n","#prints the number of tokens\n","average_tokens = round(len(words)/len(sents))\n","print(\"The average number of tokens per sentence is\",\n","average_tokens) \n","#prints the average number of tokens per sentence\n","unique_tokens = set(words)\n","print(\"The number of unique tokens are\", len(unique_tokens)) \n","#prints the number of unique tokens"],"execution_count":6,"outputs":[{"output_type":"stream","text":["The number of sentences is 96\n","The number of tokens is 2803\n","The average number of tokens per sentence is 29\n","The number of unique tokens are 909\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t8edwJf2ZiJr","executionInfo":{"status":"ok","timestamp":1621598610774,"user_tz":-270,"elapsed":24,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"530c1681-8409-4104-cde0-2dd692ba89ea"},"source":["from nltk.util import ngrams\n","from nltk.corpus import stopwords\n","stop_words = set(stopwords.words('english'))\n","unigram=[]\n","bigram=[]\n","trigram=[]\n","fourgram=[]\n","tokenized_text = []\n","for sentence in sents:\n","    sentence = sentence.lower()\n","    sequence = word_tokenize(sentence) \n","    for word in sequence:\n","        if (word =='.'):\n","            sequence.remove(word) \n","        else:\n","            unigram.append(word)\n","    tokenized_text.append(sequence) \n","    bigram.extend(list(ngrams(sequence, 2)))  \n","#unigram, bigram, trigram, and fourgram models are created\n","    trigram.extend(list(ngrams(sequence, 3)))\n","    fourgram.extend(list(ngrams(sequence, 4)))\n","def removal(x):     \n","#removes ngrams containing only stopwords\n","    y = []\n","    for pair in x:\n","        count = 0\n","        for word in pair:\n","            if word in stop_words:\n","                count = count or 0\n","            else:\n","                count = count or 1\n","        if (count==1):\n","            y.append(pair)\n","    return(y)\n","bigram = removal(bigram)\n","trigram = removal(trigram)             \n","fourgram = removal(fourgram)\n","freq_bi = nltk.FreqDist(bigram)\n","freq_tri = nltk.FreqDist(trigram)\n","freq_four = nltk.FreqDist(fourgram)\n","print(\"Most common n-grams without stopword removal and without add-1 smoothing: \\n\")\n","print (\"Most common bigrams: \", freq_bi.most_common(5))\n","print (\"\\nMost common trigrams: \", freq_tri.most_common(5))\n","print (\"\\nMost common fourgrams: \", freq_four.most_common(5))"],"execution_count":7,"outputs":[{"output_type":"stream","text":["Most common n-grams without stopword removal and without add-1 smoothing: \n","\n","Most common bigrams:  [(('کرنل', 'لينوکس'), 25), (('دانشگاه', 'مينه\\u200cسوتا'), 21), (('از', 'اين'), 12), (('را', 'به'), 11), (('در', 'مورد'), 10)]\n","\n","Most common trigrams:  [(('پروژه\\u200cي', 'کرنل', 'لينوکس'), 5), (('کرنل', 'لينوکس', 'و'), 4), (('به', 'کرنل', 'لينوکس'), 4), (('کرنل', 'لينوکس', 'را'), 3), (('پروژه\\u200cهاي', 'متن', 'باز'), 3)]\n","\n","Most common fourgrams:  [(('راه', 'درازي', 'در', 'پيش'), 3), (('روز', 'ششم', 'آوريل', '،'), 2), (('کرنل', 'لينوکس', 'را', 'به'), 2), (('کروآ', 'هارتمن،', 'يکي', 'از'), 2), (('اينجا', 'ديگر', 'جاي', 'شما'), 2)]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SFIzb9wzZ19d","executionInfo":{"status":"ok","timestamp":1621598610775,"user_tz":-270,"elapsed":21,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"feb77568-2e83-4c1a-ff27-c89144bac1e7"},"source":["#stopwords = code for downloading stop words through nltk\n","from nltk.corpus import stopwords\n","stop_words = set(stopwords.words('english'))\n","#prints top 10 unigrams, bigrams after removing stopwords\n","print(\"Most common n-grams with stopword removal and without add-1 smoothing: \\n\")\n","unigram_sw_removed = [p for p in unigram if p not in stop_words]\n","fdist = nltk.FreqDist(unigram_sw_removed)\n","print(\"Most common unigrams: \", fdist.most_common(10))\n","bigram_sw_removed = []\n","bigram_sw_removed.extend(list(ngrams(unigram_sw_removed, 2)))\n","fdist = nltk.FreqDist(bigram_sw_removed)\n","print(\"\\nMost common bigrams: \", fdist.most_common(10))"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Most common n-grams with stopword removal and without add-1 smoothing: \n","\n","Most common unigrams:  [('اين', 95), ('از', 84), ('به', 80), ('در', 68), ('و', 67), ('را', 54), ('که', 44), ('لينوکس', 42), ('دانشگاه', 41), ('است', 38)]\n","\n","Most common bigrams:  [(('کرنل', 'لينوکس'), 25), (('دانشگاه', 'مينه\\u200cسوتا'), 21), (('از', 'اين'), 12), (('را', 'به'), 11), (('در', 'مورد'), 10), (('به', 'اين'), 9), (('در', 'اين'), 7), (('اين', 'دانشگاه'), 7), (('سيستم', 'عامل'), 7), (('متن', 'باز'), 7)]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"erTrdH5_Z4y4","executionInfo":{"status":"ok","timestamp":1621598611372,"user_tz":-270,"elapsed":611,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}}},"source":["#Add-1 smoothing is performed here.\n","            \n","ngrams_all = {1:[], 2:[], 3:[], 4:[]}\n","for i in range(4):\n","    for each in tokenized_text:\n","        for j in ngrams(each, i+1):\n","            ngrams_all[i+1].append(j);\n","ngrams_voc = {1:set([]), 2:set([]), 3:set([]), 4:set([])}\n","for i in range(4):\n","    for gram in ngrams_all[i+1]:\n","        if gram not in ngrams_voc[i+1]:\n","            ngrams_voc[i+1].add(gram)\n","total_ngrams = {1:-1, 2:-1, 3:-1, 4:-1}\n","total_voc = {1:-1, 2:-1, 3:-1, 4:-1}\n","for i in range(4):\n","    total_ngrams[i+1] = len(ngrams_all[i+1])\n","    total_voc[i+1] = len(ngrams_voc[i+1])                       \n","    \n","ngrams_prob = {1:[], 2:[], 3:[], 4:[]}\n","for i in range(4):\n","    for ngram in ngrams_voc[i+1]:\n","        tlist = [ngram]\n","        tlist.append(ngrams_all[i+1].count(ngram))\n","        ngrams_prob[i+1].append(tlist)\n","    \n","for i in range(4):\n","    for ngram in ngrams_prob[i+1]:\n","        ngram[-1] = (ngram[-1]+1)/(total_ngrams[i+1]+total_voc[i+1])             #add-1 smoothing"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QFWfVoLuZ-4v","executionInfo":{"status":"ok","timestamp":1621598611373,"user_tz":-270,"elapsed":10,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"7a76bcb6-a442-4b85-d17a-387a372b2001"},"source":["#Prints top 10 unigram, bigram, trigram, fourgram after smoothing\n","print(\"Most common n-grams without stopword removal and with add-1 smoothing: \\n\")\n","for i in range(4):\n","    ngrams_prob[i+1] = sorted(ngrams_prob[i+1], key = lambda x:x[1], reverse = True)\n","    \n","print (\"Most common unigrams: \", str(ngrams_prob[1][:10]))\n","print (\"\\nMost common bigrams: \", str(ngrams_prob[2][:10]))\n","print (\"\\nMost common trigrams: \", str(ngrams_prob[3][:10]))\n","print (\"\\nMost common fourgrams: \", str(ngrams_prob[4][:10]))"],"execution_count":10,"outputs":[{"output_type":"stream","text":["Most common n-grams without stopword removal and with add-1 smoothing: \n","\n","Most common unigrams:  [[('اين',), 0.026556016597510373], [('از',), 0.02351313969571231], [('به',), 0.022406639004149378], [('در',), 0.019087136929460582], [('و',), 0.018810511756569847], [('را',), 0.015214384508990318], [('که',), 0.012448132780082987], [('لينوکس',), 0.01189488243430152], [('دانشگاه',), 0.011618257261410789], [('است',), 0.010788381742738589]]\n","\n","Most common bigrams:  [[('کرنل', 'لينوکس'), 0.005491024287222809], [('دانشگاه', 'مينه\\u200cسوتا'), 0.004646251319957761], [('از', 'اين'), 0.0027455121436114043], [('را', 'به'), 0.0025343189017951427], [('در', 'مورد'), 0.0023231256599788807], [('به', 'اين'), 0.0021119324181626186], [('در', 'اين'), 0.001689545934530095], [('اين', 'دانشگاه'), 0.001689545934530095], [('سيستم', 'عامل'), 0.001689545934530095], [('متن', 'باز'), 0.001689545934530095]]\n","\n","Most common trigrams:  [[('پروژه\\u200cي', 'کرنل', 'لينوکس'), 0.0012165450121654502], [('به', 'کرنل', 'لينوکس'), 0.001013787510137875], [('کرنل', 'لينوکس', 'و'), 0.001013787510137875], [('درازي', 'در', 'پيش'), 0.0008110300081103001], [('پروژه\\u200cهاي', 'متن', 'باز'), 0.0008110300081103001], [('کرنل', 'لينوکس', 'را'), 0.0008110300081103001], [('دانشگاه', 'مينه\\u200cسوتا', 'براي'), 0.0008110300081103001], [('براي', 'جلب', 'اعتماد'), 0.0008110300081103001], [('انجمن', 'کرنل', 'لينوکس'), 0.0008110300081103001], [('نگهدارندگان', 'کرنل', 'لينوکس'), 0.0008110300081103001]]\n","\n","Most common fourgrams:  [[('راه', 'درازي', 'در', 'پيش'), 0.0008319467554076539], [('را', 'به', 'کرنل', 'لينوکس'), 0.0006239600665557404], [('نگهدارندگان', 'کرنل', 'لينوکس', 'و'), 0.0006239600665557404], [('ايميل\\u200cهاي', 'وابسته', 'به', 'دانشگاه'), 0.0006239600665557404], [('مورد', 'از', 'اين', 'باگ\\u200cها'), 0.0006239600665557404], [('وابسته', 'به', 'دانشگاه', 'مينه\\u200cسوتا'), 0.0006239600665557404], [('براي', 'جلب', 'اعتماد', 'تيم'), 0.0006239600665557404], [('داستان', 'از', 'سال', 'شروع'), 0.0006239600665557404], [('اينجا', 'ديگر', 'جاي', 'شما'), 0.0006239600665557404], [('روز', 'ششم', 'آوريل', '،'), 0.0006239600665557404]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"4EGVlDwDeIAR","executionInfo":{"status":"ok","timestamp":1621598611374,"user_tz":-270,"elapsed":8,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}}},"source":["testset = \"دانشگاه مينه سوتا در زمينه توسعه سيستم عامل لينوکس توزيع کرنل در حال همکاري بود اما در حين همکاري و ايجاد کد هاي جديد يک محافظ و نگهدارنده سيستم کرنل لينوکس به محض مشاهده کد هاي دانشگاه مينه سوتا متوجه خرابکاري از سوي آن ها شد اين کد ها که توسط توسعه دهندگان در داخل اين دانشگاه نوشته شده بود داراي نقض هاي امنيتي بودند  اما با توجه به افشاي اين موضوع محققان دانشگاه مينه سوتا درباره اين موضوع گفتند که هدف از راه اندازي و نوشتن چنين کد ها اين بوده است که چگونه سيستم و کد مخرب مي‌تواند از داخل کرنل لينوکس عبور کند و يک سيستم را آلوده نمايد\"\n","\n"],"execution_count":11,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"HZnVTn3t1SQM"},"source":["PERRRRPLEXITY"]},{"cell_type":"code","metadata":{"id":"h-5_1i57d5jn","executionInfo":{"status":"ok","timestamp":1621598631019,"user_tz":-270,"elapsed":398,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}}},"source":["#perplexity\n","#computes perplexity of the unigram model on a testset  \n","def perplexity(testset, model):\n","    testset = testset.split()\n","    perplexity = 1\n","    N = 0\n","    for word in testset:\n","        N += 1\n","        perplexity = perplexity * (1/model[word])\n","    perplexity = pow(perplexity, 1/float(N)) \n","    return perplexity"],"execution_count":12,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":292},"id":"twREmQEoz4UA","executionInfo":{"status":"error","timestamp":1621598772415,"user_tz":-270,"elapsed":447,"user":{"displayName":"Amir Dodangeh","photoUrl":"","userId":"03691380656141451894"}},"outputId":"29db5952-4b92-4dae-e3e4-5752841c16a5"},"source":["print (perplexity(testset,fdist))"],"execution_count":14,"outputs":[{"output_type":"error","ename":"ZeroDivisionError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-14-e0fa9a231d1d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mperplexity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtestset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfdist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m<ipython-input-12-44412d8115a9>\u001b[0m in \u001b[0;36mperplexity\u001b[0;34m(testset, model)\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtestset\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mN\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mperplexity\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mperplexity\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mperplexity\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mperplexity\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mperplexity\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"]}]}]}