Hate Speech Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Relegious Hate speech classifier using Natural Language processing(Naive bayes classifier)\n",
    "By-Archit P. Hadge\n",
    "B.Tech CSE VIT Pune(2018-2022)\n",
    "'''\n",
    "import GetOldTweets3 as got"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "#A function that returns list of tweets with given keyword of given quantity\n",
    "def buildTestSet(keyword,quantity):\n",
    "    tweetCriteria = got.manager.TweetCriteria().setQuerySearch(keyword).setMaxTweets(quantity)\n",
    "    tweets = got.manager.TweetManager.getTweets(tweetCriteria)\n",
    "    tweets=[{\"text\":tweet.text,\"label\":None,'link':tweet.permalink} for tweet in tweets]\n",
    "    return tweets\n",
    "\n",
    "#Function to get tweets of given keyword and provided label\n",
    "def buildTrainSetSegment(keyword,quantity,label):\n",
    "    trainSetSeg=[]\n",
    "    tweetCriteria = got.manager.TweetCriteria().setQuerySearch(keyword).setMaxTweets(quantity)\n",
    "    tweets = got.manager.TweetManager.getTweets(tweetCriteria)\n",
    "    for tweet in tweets:\n",
    "        obj={}\n",
    "        obj[\"topic\"]=keyword\n",
    "        obj[\"tweet_id\"]=tweet.id\n",
    "        obj[\"label\"]=label\n",
    "        obj[\"text\"]=tweet.text\n",
    "        obj[\"link\"]=tweet.permalink\n",
    "        trainSetSeg.append(obj)\n",
    "    return trainSetSeg\n",
    "\n",
    "import csv\n",
    "#To create csv file containing tweets info\n",
    "def buildCSV(filename,tweets):\n",
    "    csvfile=open(filename,'a')\n",
    "    linewriter = csv.writer(csvfile,delimiter=',',quotechar=\"\\\"\")\n",
    "    for tweet in tweets:\n",
    "        linewriter.writerow([tweet[\"topic\"], tweet[\"tweet_id\"], tweet[\"label\"], tweet[\"text\"],tweet[\"link\"]])\n",
    "\n",
    "#to read created csv file and return data from it\n",
    "def readCSV(filename):\n",
    "    tweets=[]\n",
    "    csvfile=open(filename)\n",
    "    lineReader = csv.reader(csvfile,delimiter=',', quotechar=\"\\\"\")\n",
    "    for row in lineReader:\n",
    "        #print(row[4])\n",
    "        tweets.append({\"topic\":row[0],\"tweet_id\":row[1],\"label\":row[2],\"text\":row[3],\"link\":row[4]})\n",
    "    return tweets\n",
    "\n",
    "#Takes params as filename,list of keywords,total quantity and label and write it in file\n",
    "def makeDataset(file,keywords,quantity,label):\n",
    "    for i in keywords:\n",
    "        buildCSV(file,buildTrainSetSegment(i,quantity//len(keywords),label))\n",
    "\n",
    "\n",
    "l1=['jihad','kafir','allah','radical','kashmir','secularism','bharat tere tukde honge','love jihad','rohingya']\n",
    "l1=['\"'+x+'\"' for x in l1]\n",
    "#print(l)\n",
    "l2=['table','business','goverment','defence','sales','finance','trade','music','history','art']\n",
    "l2=['\"'+x+'\"' for x in l2]\n",
    "makeDataset('demo.csv',l1,4000,'hate')\n",
    "makeDataset('demo.csv',l2,3000,'normal')\n",
    "print('done')\n",
    "\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''keywords=['jihad','kafir','allah','radical','kashmir','secularism','bharat tere tukde honge','love jihad','ram-mandir','jnu','liberals','islamophobia','hinduphobia','rohingya']\n",
    "Train=[]\n",
    "print(len(Train))\n",
    "for x in keywords:\n",
    "    Train=Train+buildTrainSetSegment(x,100,'positive')\n",
    "keywords=['table','business','goverment','defence','sales','finance','trade','music','history','art']\n",
    "for x in keywords:\n",
    "    Train=Train+buildTrainSetSegment(x,100,'neutral')\n",
    "print(len(Train))\n",
    "\n",
    "Test=[]\n",
    "'''\n",
    "\n",
    "#reterive data from file 'demo.csv'\n",
    "Train=readCSV('demo.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for preprocesssing and vectorising\n",
    "import re\n",
    "from nltk.tokenize import word_tokenize\n",
    "from string import punctuation \n",
    "from nltk.corpus import stopwords \n",
    "\n",
    "class PreProcessTweets:\n",
    "    def __init__(self):\n",
    "        self._stopwords = set(stopwords.words('english') + list(punctuation) + ['AT_USER','URL'])\n",
    "        \n",
    "    def processTweets(self, list_of_tweets):\n",
    "        processedTweets=[]\n",
    "        for tweet in list_of_tweets:\n",
    "            processedTweets.append((self._processTweet(tweet[\"text\"]),tweet[\"label\"]))\n",
    "        return processedTweets\n",
    "    \n",
    "    def _processTweet(self, tweet):\n",
    "        tweet = tweet.lower() # convert text to lower-case\n",
    "        tweet = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+))', 'URL', tweet) # remove URLs\n",
    "        tweet = re.sub('@[^\\s]+', 'AT_USER', tweet) # remove usernames\n",
    "        tweet = re.sub(r'#([^\\s]+)', r'\\1', tweet) # remove the # in #hashtag\n",
    "        tweet = word_tokenize(tweet) # remove repeated characters (helloooooooo into hello)\n",
    "        return [word for word in tweet if word not in self._stopwords]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweetProcessor = PreProcessTweets()\n",
    "preprocessedTrainingSet = tweetProcessor.processTweets(Train)\n",
    "#preprocessedTestSet = tweetProcessor.processTweets(testDataSet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk \n",
    "\n",
    "def buildVocabulary(preprocessedTrainingData):\n",
    "    all_words = []\n",
    "    \n",
    "    for (words, sentiment) in preprocessedTrainingData:\n",
    "        all_words.extend(words)\n",
    "\n",
    "    wordlist = nltk.FreqDist(all_words)\n",
    "    word_features = wordlist.keys()\n",
    "    \n",
    "    return word_features\n",
    "\n",
    "def extract_features(tweet):\n",
    "    tweet_words = set(tweet)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['contains(%s)' % word] = (word in tweet_words)\n",
    "    return features \n",
    "\n",
    "word_features = buildVocabulary(preprocessedTrainingSet)\n",
    "trainingFeatures = nltk.classify.apply_features(extract_features, preprocessedTrainingSet)\n",
    "NBayesClassifier = nltk.NaiveBayesClassifier.train(trainingFeatures)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#classify hate speeches and prints link/text of the tweet\n",
    "def classify(tweets):\n",
    "    copy=tweets\n",
    "    tweets = tweetProcessor.processTweets(tweets)\n",
    "    predictions=[NBayesClassifier.classify(extract_features(tweet[0])) for tweet in tweets]\n",
    "    for i in range(len(copy)):\n",
    "        if predictions[i]=='normal':\n",
    "            print(copy[i]['link'],\"-->\",copy[i]['text'])\n",
    "\n",
    "p1=buildTestSet('market',10)\n",
    "#print(p1)\n",
    "classify(p1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}