Data Cleaning and Lemmatization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "24f38f8e",
   "metadata": {},
   "source": [
    "# Part 2: Data Cleaning, Lemmatization and Feature Extraction\n",
    "\n",
    "### In this section, the data will be cleaned to remove undesirable information for NLP, taking into account the findings in the previous part (EDA). Specifically, the text will be converted into its base form, with special chars, links, and other meaningless information removed, so that the text can be effectively vectorized/transformed for the ML models in the following sections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07e3bb0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install pandas_profiling\n",
    "!{sys.executable} -m pip install spacy\n",
    "!{sys.executable} -m pip install nltk\n",
    "!{sys.executable} -m pip install imblearn\n",
    "!{sys.executable} -m spacy download en_core_web_md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a2e1f2d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas_profiling\n",
    "import seaborn as sb\n",
    "import spacy\n",
    "import nltk\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from imblearn.over_sampling import ADASYN\n",
    "\n",
    "adasyn=ADASYN()\n",
    "sb.set()\n",
    "\n",
    "\"\"\" NOTE: SET YOUR PROJECT ROOT DIRECTORY HERE \"\"\"\n",
    "PROJ_DIR = r\"\"\n",
    "RANDOM_SPLIT_SEED = 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "308d5e84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>job_id</th>\n",
       "      <th>title</th>\n",
       "      <th>location</th>\n",
       "      <th>department</th>\n",
       "      <th>salary_range</th>\n",
       "      <th>company_profile</th>\n",
       "      <th>description</th>\n",
       "      <th>requirements</th>\n",
       "      <th>benefits</th>\n",
       "      <th>telecommuting</th>\n",
       "      <th>has_company_logo</th>\n",
       "      <th>has_questions</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>required_experience</th>\n",
       "      <th>required_education</th>\n",
       "      <th>industry</th>\n",
       "      <th>function</th>\n",
       "      <th>fraudulent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Marketing Intern</td>\n",
       "      <td>US, NY, New York</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>NaN</td>\n",
       "      <td>We're Food52, and we've created a groundbreaki...</td>\n",
       "      <td>Food52, a fast-growing, James Beard Award-winn...</td>\n",
       "      <td>Experience with content management systems a m...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Other</td>\n",
       "      <td>Internship</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Customer Service - Cloud Video Production</td>\n",
       "      <td>NZ, , Auckland</td>\n",
       "      <td>Success</td>\n",
       "      <td>NaN</td>\n",
       "      <td>90 Seconds, the worlds Cloud Video Production ...</td>\n",
       "      <td>Organised - Focused - Vibrant - Awesome!Do you...</td>\n",
       "      <td>What we expect from you:Your key responsibilit...</td>\n",
       "      <td>What you will get from usThrough being part of...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Marketing and Advertising</td>\n",
       "      <td>Customer Service</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Commissioning Machinery Assistant (CMA)</td>\n",
       "      <td>US, IA, Wever</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Valor Services provides Workforce Solutions th...</td>\n",
       "      <td>Our client, located in Houston, is actively se...</td>\n",
       "      <td>Implement pre-commissioning and commissioning ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Account Executive - Washington DC</td>\n",
       "      <td>US, DC, Washington</td>\n",
       "      <td>Sales</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our passion for improving quality of life thro...</td>\n",
       "      <td>THE COMPANY: ESRI – Environmental Systems Rese...</td>\n",
       "      <td>EDUCATION: Bachelor’s or Master’s in GIS, busi...</td>\n",
       "      <td>Our culture is anything but corporate—we have ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>Computer Software</td>\n",
       "      <td>Sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Bill Review Manager</td>\n",
       "      <td>US, FL, Fort Worth</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SpotSource Solutions LLC is a Global Human Cap...</td>\n",
       "      <td>JOB TITLE: Itemization Review ManagerLOCATION:...</td>\n",
       "      <td>QUALIFICATIONS:RN license in the State of Texa...</td>\n",
       "      <td>Full Benefits Offered</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>Hospital &amp; Health Care</td>\n",
       "      <td>Health Care Provider</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   job_id                                      title            location  \\\n",
       "0       1                           Marketing Intern    US, NY, New York   \n",
       "1       2  Customer Service - Cloud Video Production      NZ, , Auckland   \n",
       "2       3    Commissioning Machinery Assistant (CMA)       US, IA, Wever   \n",
       "3       4          Account Executive - Washington DC  US, DC, Washington   \n",
       "4       5                        Bill Review Manager  US, FL, Fort Worth   \n",
       "\n",
       "  department salary_range                                    company_profile  \\\n",
       "0  Marketing          NaN  We're Food52, and we've created a groundbreaki...   \n",
       "1    Success          NaN  90 Seconds, the worlds Cloud Video Production ...   \n",
       "2        NaN          NaN  Valor Services provides Workforce Solutions th...   \n",
       "3      Sales          NaN  Our passion for improving quality of life thro...   \n",
       "4        NaN          NaN  SpotSource Solutions LLC is a Global Human Cap...   \n",
       "\n",
       "                                         description  \\\n",
       "0  Food52, a fast-growing, James Beard Award-winn...   \n",
       "1  Organised - Focused - Vibrant - Awesome!Do you...   \n",
       "2  Our client, located in Houston, is actively se...   \n",
       "3  THE COMPANY: ESRI – Environmental Systems Rese...   \n",
       "4  JOB TITLE: Itemization Review ManagerLOCATION:...   \n",
       "\n",
       "                                        requirements  \\\n",
       "0  Experience with content management systems a m...   \n",
       "1  What we expect from you:Your key responsibilit...   \n",
       "2  Implement pre-commissioning and commissioning ...   \n",
       "3  EDUCATION: Bachelor’s or Master’s in GIS, busi...   \n",
       "4  QUALIFICATIONS:RN license in the State of Texa...   \n",
       "\n",
       "                                            benefits  telecommuting  \\\n",
       "0                                                NaN              0   \n",
       "1  What you will get from usThrough being part of...              0   \n",
       "2                                                NaN              0   \n",
       "3  Our culture is anything but corporate—we have ...              0   \n",
       "4                              Full Benefits Offered              0   \n",
       "\n",
       "   has_company_logo  has_questions employment_type required_experience  \\\n",
       "0                 1              0           Other          Internship   \n",
       "1                 1              0       Full-time      Not Applicable   \n",
       "2                 1              0             NaN                 NaN   \n",
       "3                 1              0       Full-time    Mid-Senior level   \n",
       "4                 1              1       Full-time    Mid-Senior level   \n",
       "\n",
       "  required_education                   industry              function  \\\n",
       "0                NaN                        NaN             Marketing   \n",
       "1                NaN  Marketing and Advertising      Customer Service   \n",
       "2                NaN                        NaN                   NaN   \n",
       "3  Bachelor's Degree          Computer Software                 Sales   \n",
       "4  Bachelor's Degree     Hospital & Health Care  Health Care Provider   \n",
       "\n",
       "   fraudulent  \n",
       "0           0  \n",
       "1           0  \n",
       "2           0  \n",
       "3           0  \n",
       "4           0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(PROJ_DIR + r\"\\sc1015-project\\dataset\\fake_job_postings.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8d551bd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in df.columns:\n",
    "    df[column]=df[column].fillna(f'missing_{column}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "889bd1cd",
   "metadata": {},
   "source": [
    "#### We observe that there are 4 boolean valued columns (including `fraudulent`). However, they are stored as int64 and will need to be encoded into text for the nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cff35771",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>job_id</th>\n",
       "      <th>title</th>\n",
       "      <th>location</th>\n",
       "      <th>department</th>\n",
       "      <th>salary_range</th>\n",
       "      <th>company_profile</th>\n",
       "      <th>description</th>\n",
       "      <th>requirements</th>\n",
       "      <th>benefits</th>\n",
       "      <th>telecommuting</th>\n",
       "      <th>has_company_logo</th>\n",
       "      <th>has_questions</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>required_experience</th>\n",
       "      <th>required_education</th>\n",
       "      <th>industry</th>\n",
       "      <th>function</th>\n",
       "      <th>fraudulent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Marketing Intern</td>\n",
       "      <td>US, NY, New York</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>We're Food52, and we've created a groundbreaki...</td>\n",
       "      <td>Food52, a fast-growing, James Beard Award-winn...</td>\n",
       "      <td>Experience with content management systems a m...</td>\n",
       "      <td>missing_benefits</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>Other</td>\n",
       "      <td>Internship</td>\n",
       "      <td>missing_required_education</td>\n",
       "      <td>missing_industry</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Customer Service - Cloud Video Production</td>\n",
       "      <td>NZ, , Auckland</td>\n",
       "      <td>Success</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>90 Seconds, the worlds Cloud Video Production ...</td>\n",
       "      <td>Organised - Focused - Vibrant - Awesome!Do you...</td>\n",
       "      <td>What we expect from you:Your key responsibilit...</td>\n",
       "      <td>What you will get from usThrough being part of...</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>missing_required_education</td>\n",
       "      <td>Marketing and Advertising</td>\n",
       "      <td>Customer Service</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Commissioning Machinery Assistant (CMA)</td>\n",
       "      <td>US, IA, Wever</td>\n",
       "      <td>missing_department</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>Valor Services provides Workforce Solutions th...</td>\n",
       "      <td>Our client, located in Houston, is actively se...</td>\n",
       "      <td>Implement pre-commissioning and commissioning ...</td>\n",
       "      <td>missing_benefits</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>missing_employment_type</td>\n",
       "      <td>missing_required_experience</td>\n",
       "      <td>missing_required_education</td>\n",
       "      <td>missing_industry</td>\n",
       "      <td>missing_function</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Account Executive - Washington DC</td>\n",
       "      <td>US, DC, Washington</td>\n",
       "      <td>Sales</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>Our passion for improving quality of life thro...</td>\n",
       "      <td>THE COMPANY: ESRI – Environmental Systems Rese...</td>\n",
       "      <td>EDUCATION: Bachelor’s or Master’s in GIS, busi...</td>\n",
       "      <td>Our culture is anything but corporate—we have ...</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>Computer Software</td>\n",
       "      <td>Sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Bill Review Manager</td>\n",
       "      <td>US, FL, Fort Worth</td>\n",
       "      <td>missing_department</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>SpotSource Solutions LLC is a Global Human Cap...</td>\n",
       "      <td>JOB TITLE: Itemization Review ManagerLOCATION:...</td>\n",
       "      <td>QUALIFICATIONS:RN license in the State of Texa...</td>\n",
       "      <td>Full Benefits Offered</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>yes_has_questions</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>Hospital &amp; Health Care</td>\n",
       "      <td>Health Care Provider</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   job_id                                      title            location  \\\n",
       "0       1                           Marketing Intern    US, NY, New York   \n",
       "1       2  Customer Service - Cloud Video Production      NZ, , Auckland   \n",
       "2       3    Commissioning Machinery Assistant (CMA)       US, IA, Wever   \n",
       "3       4          Account Executive - Washington DC  US, DC, Washington   \n",
       "4       5                        Bill Review Manager  US, FL, Fort Worth   \n",
       "\n",
       "           department          salary_range  \\\n",
       "0           Marketing  missing_salary_range   \n",
       "1             Success  missing_salary_range   \n",
       "2  missing_department  missing_salary_range   \n",
       "3               Sales  missing_salary_range   \n",
       "4  missing_department  missing_salary_range   \n",
       "\n",
       "                                     company_profile  \\\n",
       "0  We're Food52, and we've created a groundbreaki...   \n",
       "1  90 Seconds, the worlds Cloud Video Production ...   \n",
       "2  Valor Services provides Workforce Solutions th...   \n",
       "3  Our passion for improving quality of life thro...   \n",
       "4  SpotSource Solutions LLC is a Global Human Cap...   \n",
       "\n",
       "                                         description  \\\n",
       "0  Food52, a fast-growing, James Beard Award-winn...   \n",
       "1  Organised - Focused - Vibrant - Awesome!Do you...   \n",
       "2  Our client, located in Houston, is actively se...   \n",
       "3  THE COMPANY: ESRI – Environmental Systems Rese...   \n",
       "4  JOB TITLE: Itemization Review ManagerLOCATION:...   \n",
       "\n",
       "                                        requirements  \\\n",
       "0  Experience with content management systems a m...   \n",
       "1  What we expect from you:Your key responsibilit...   \n",
       "2  Implement pre-commissioning and commissioning ...   \n",
       "3  EDUCATION: Bachelor’s or Master’s in GIS, busi...   \n",
       "4  QUALIFICATIONS:RN license in the State of Texa...   \n",
       "\n",
       "                                            benefits     telecommuting  \\\n",
       "0                                   missing_benefits  no_telecommuting   \n",
       "1  What you will get from usThrough being part of...  no_telecommuting   \n",
       "2                                   missing_benefits  no_telecommuting   \n",
       "3  Our culture is anything but corporate—we have ...  no_telecommuting   \n",
       "4                              Full Benefits Offered  no_telecommuting   \n",
       "\n",
       "       has_company_logo      has_questions          employment_type  \\\n",
       "0  yes_has_company_logo   no_has_questions                    Other   \n",
       "1  yes_has_company_logo   no_has_questions                Full-time   \n",
       "2  yes_has_company_logo   no_has_questions  missing_employment_type   \n",
       "3  yes_has_company_logo   no_has_questions                Full-time   \n",
       "4  yes_has_company_logo  yes_has_questions                Full-time   \n",
       "\n",
       "           required_experience          required_education  \\\n",
       "0                   Internship  missing_required_education   \n",
       "1               Not Applicable  missing_required_education   \n",
       "2  missing_required_experience  missing_required_education   \n",
       "3             Mid-Senior level           Bachelor's Degree   \n",
       "4             Mid-Senior level           Bachelor's Degree   \n",
       "\n",
       "                    industry              function  fraudulent  \n",
       "0           missing_industry             Marketing           0  \n",
       "1  Marketing and Advertising      Customer Service           0  \n",
       "2           missing_industry      missing_function           0  \n",
       "3          Computer Software                 Sales           0  \n",
       "4     Hospital & Health Care  Health Care Provider           0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boolean_cols = ['has_company_logo','has_questions','telecommuting']\n",
    "\n",
    "# changing boolean values to text and appending their column names\n",
    "for col in boolean_cols:\n",
    "    df.loc[df[col]==0,col] = f'no_{col}'\n",
    "    df.loc[df[col]==1,col] = f'yes_{col}'\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de0fe6fc",
   "metadata": {},
   "source": [
    "#### Similarly, categorical values will be modified to the format of the boolean values (i.e {data value}_{col name})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "093a8375",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>job_id</th>\n",
       "      <th>title</th>\n",
       "      <th>location</th>\n",
       "      <th>department</th>\n",
       "      <th>salary_range</th>\n",
       "      <th>company_profile</th>\n",
       "      <th>description</th>\n",
       "      <th>requirements</th>\n",
       "      <th>benefits</th>\n",
       "      <th>telecommuting</th>\n",
       "      <th>has_company_logo</th>\n",
       "      <th>has_questions</th>\n",
       "      <th>employment_type</th>\n",
       "      <th>required_experience</th>\n",
       "      <th>required_education</th>\n",
       "      <th>industry</th>\n",
       "      <th>function</th>\n",
       "      <th>fraudulent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Marketing Intern</td>\n",
       "      <td>US, NY, New York</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>We're Food52, and we've created a groundbreaki...</td>\n",
       "      <td>Food52, a fast-growing, James Beard Award-winn...</td>\n",
       "      <td>Experience with content management systems a m...</td>\n",
       "      <td>missing_benefits</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>Other_employment_type</td>\n",
       "      <td>Internship_required_experience</td>\n",
       "      <td>missing_required_education_required_education</td>\n",
       "      <td>missing_industry</td>\n",
       "      <td>Marketing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Customer Service - Cloud Video Production</td>\n",
       "      <td>NZ, , Auckland</td>\n",
       "      <td>Success</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>90 Seconds, the worlds Cloud Video Production ...</td>\n",
       "      <td>Organised - Focused - Vibrant - Awesome!Do you...</td>\n",
       "      <td>What we expect from you:Your key responsibilit...</td>\n",
       "      <td>What you will get from usThrough being part of...</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>Full_time_employment_type</td>\n",
       "      <td>Not_Applicable_required_experience</td>\n",
       "      <td>missing_required_education_required_education</td>\n",
       "      <td>Marketing and Advertising</td>\n",
       "      <td>Customer Service</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Commissioning Machinery Assistant (CMA)</td>\n",
       "      <td>US, IA, Wever</td>\n",
       "      <td>missing_department</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>Valor Services provides Workforce Solutions th...</td>\n",
       "      <td>Our client, located in Houston, is actively se...</td>\n",
       "      <td>Implement pre-commissioning and commissioning ...</td>\n",
       "      <td>missing_benefits</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>missing_employment_type_employment_type</td>\n",
       "      <td>missing_required_experience_required_experience</td>\n",
       "      <td>missing_required_education_required_education</td>\n",
       "      <td>missing_industry</td>\n",
       "      <td>missing_function</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Account Executive - Washington DC</td>\n",
       "      <td>US, DC, Washington</td>\n",
       "      <td>Sales</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>Our passion for improving quality of life thro...</td>\n",
       "      <td>THE COMPANY: ESRI – Environmental Systems Rese...</td>\n",
       "      <td>EDUCATION: Bachelor’s or Master’s in GIS, busi...</td>\n",
       "      <td>Our culture is anything but corporate—we have ...</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>no_has_questions</td>\n",
       "      <td>Full_time_employment_type</td>\n",
       "      <td>Mid_Senior_level_required_experience</td>\n",
       "      <td>Bachelor's_Degree_required_education</td>\n",
       "      <td>Computer Software</td>\n",
       "      <td>Sales</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Bill Review Manager</td>\n",
       "      <td>US, FL, Fort Worth</td>\n",
       "      <td>missing_department</td>\n",
       "      <td>missing_salary_range</td>\n",
       "      <td>SpotSource Solutions LLC is a Global Human Cap...</td>\n",
       "      <td>JOB TITLE: Itemization Review ManagerLOCATION:...</td>\n",
       "      <td>QUALIFICATIONS:RN license in the State of Texa...</td>\n",
       "      <td>Full Benefits Offered</td>\n",
       "      <td>no_telecommuting</td>\n",
       "      <td>yes_has_company_logo</td>\n",
       "      <td>yes_has_questions</td>\n",
       "      <td>Full_time_employment_type</td>\n",
       "      <td>Mid_Senior_level_required_experience</td>\n",
       "      <td>Bachelor's_Degree_required_education</td>\n",
       "      <td>Hospital &amp; Health Care</td>\n",
       "      <td>Health Care Provider</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   job_id                                      title            location  \\\n",
       "0       1                           Marketing Intern    US, NY, New York   \n",
       "1       2  Customer Service - Cloud Video Production      NZ, , Auckland   \n",
       "2       3    Commissioning Machinery Assistant (CMA)       US, IA, Wever   \n",
       "3       4          Account Executive - Washington DC  US, DC, Washington   \n",
       "4       5                        Bill Review Manager  US, FL, Fort Worth   \n",
       "\n",
       "           department          salary_range  \\\n",
       "0           Marketing  missing_salary_range   \n",
       "1             Success  missing_salary_range   \n",
       "2  missing_department  missing_salary_range   \n",
       "3               Sales  missing_salary_range   \n",
       "4  missing_department  missing_salary_range   \n",
       "\n",
       "                                     company_profile  \\\n",
       "0  We're Food52, and we've created a groundbreaki...   \n",
       "1  90 Seconds, the worlds Cloud Video Production ...   \n",
       "2  Valor Services provides Workforce Solutions th...   \n",
       "3  Our passion for improving quality of life thro...   \n",
       "4  SpotSource Solutions LLC is a Global Human Cap...   \n",
       "\n",
       "                                         description  \\\n",
       "0  Food52, a fast-growing, James Beard Award-winn...   \n",
       "1  Organised - Focused - Vibrant - Awesome!Do you...   \n",
       "2  Our client, located in Houston, is actively se...   \n",
       "3  THE COMPANY: ESRI – Environmental Systems Rese...   \n",
       "4  JOB TITLE: Itemization Review ManagerLOCATION:...   \n",
       "\n",
       "                                        requirements  \\\n",
       "0  Experience with content management systems a m...   \n",
       "1  What we expect from you:Your key responsibilit...   \n",
       "2  Implement pre-commissioning and commissioning ...   \n",
       "3  EDUCATION: Bachelor’s or Master’s in GIS, busi...   \n",
       "4  QUALIFICATIONS:RN license in the State of Texa...   \n",
       "\n",
       "                                            benefits     telecommuting  \\\n",
       "0                                   missing_benefits  no_telecommuting   \n",
       "1  What you will get from usThrough being part of...  no_telecommuting   \n",
       "2                                   missing_benefits  no_telecommuting   \n",
       "3  Our culture is anything but corporate—we have ...  no_telecommuting   \n",
       "4                              Full Benefits Offered  no_telecommuting   \n",
       "\n",
       "       has_company_logo      has_questions  \\\n",
       "0  yes_has_company_logo   no_has_questions   \n",
       "1  yes_has_company_logo   no_has_questions   \n",
       "2  yes_has_company_logo   no_has_questions   \n",
       "3  yes_has_company_logo   no_has_questions   \n",
       "4  yes_has_company_logo  yes_has_questions   \n",
       "\n",
       "                           employment_type  \\\n",
       "0                    Other_employment_type   \n",
       "1                Full_time_employment_type   \n",
       "2  missing_employment_type_employment_type   \n",
       "3                Full_time_employment_type   \n",
       "4                Full_time_employment_type   \n",
       "\n",
       "                               required_experience  \\\n",
       "0                   Internship_required_experience   \n",
       "1               Not_Applicable_required_experience   \n",
       "2  missing_required_experience_required_experience   \n",
       "3             Mid_Senior_level_required_experience   \n",
       "4             Mid_Senior_level_required_experience   \n",
       "\n",
       "                              required_education                   industry  \\\n",
       "0  missing_required_education_required_education           missing_industry   \n",
       "1  missing_required_education_required_education  Marketing and Advertising   \n",
       "2  missing_required_education_required_education           missing_industry   \n",
       "3           Bachelor's_Degree_required_education          Computer Software   \n",
       "4           Bachelor's_Degree_required_education     Hospital & Health Care   \n",
       "\n",
       "               function  fraudulent  \n",
       "0             Marketing           0  \n",
       "1      Customer Service           0  \n",
       "2      missing_function           0  \n",
       "3                 Sales           0  \n",
       "4  Health Care Provider           0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_cols = ['required_experience','required_education','employment_type']\n",
    "for col in categorical_cols:\n",
    "    val = list(df[col].values)\n",
    "    val = [\"_\".join(re.findall(r\"[\\w']+\",i)) + f\"_{col}\" for i in val]\n",
    "    df[col] = np.array(val)\n",
    "    \n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ced9cd3c",
   "metadata": {},
   "source": [
    "#### Now, the text of all the columns will be concatenated into a single block to be processed by tf-idf. The text will also be converted to lowercase to reduce variables that have little to no effect (in this case) on the meaning of the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d0497ded",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['title', 'location', 'department', 'salary_range', 'company_profile', 'description', 'requirements', 'benefits', 'telecommuting', 'has_company_logo', 'has_questions', 'employment_type', 'required_experience', 'required_education', 'industry', 'function']\n",
      "count                                                 17880\n",
      "unique                                                17596\n",
      "top        title insurance ops: sr title officer/counsel...\n",
      "freq                                                      6\n",
      "dtype: object\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0     marketing intern us, ny, new york marketing m...\n",
       "1     customer service - cloud video production nz,...\n",
       "2     commissioning machinery assistant (cma) us, i...\n",
       "3     account executive - washington dc us, dc, was...\n",
       "4     bill review manager us, fl, fort worth missin...\n",
       "dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_of_cols = list(df)\n",
    "list_of_cols.remove('fraudulent') # remove fraudulent because it is the desired classification\n",
    "list_of_cols.remove('job_id') # remove job_id because useless data\n",
    "print(list_of_cols)\n",
    "\n",
    "ser=pd.Series(np.full(shape=(df.shape[0],), fill_value=''))\n",
    "for col in list_of_cols:\n",
    "    ser += (\" \" + df[col].str.lower())  # Convert text to lowercase\n",
    "\n",
    "    \n",
    "    \n",
    "print(ser.describe())\n",
    "ser.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eac4aa73",
   "metadata": {},
   "source": [
    "#### We now have 17880 rows of text blocks, each representing a single job posting.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb62b23a",
   "metadata": {},
   "source": [
    "#### Here we will generate a list of stopwords, and exclude 'no' from it since this word encodes information regarding the boolean columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "22e60413",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Workstation\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count                                                 17880\n",
      "unique                                                17596\n",
      "top        title insurance ops: sr title officer/counsel...\n",
      "freq                                                      6\n",
      "dtype: object\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0     marketing intern us, ny, new york marketing m...\n",
       "1     customer service - cloud video production nz,...\n",
       "2     commissioning machinery assistant (cma) us, i...\n",
       "3     account executive - washington dc us, dc, was...\n",
       "4     bill review manager us, fl, fort worth missin...\n",
       "dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "list_of_stopwords = nltk.corpus.stopwords.words('english')\n",
    "list_of_stopwords.remove('no')\n",
    "\n",
    "nlp = spacy.load('en_core_web_md')\n",
    "\n",
    "\n",
    "def lemmatize_text(text):\n",
    "    text = nlp(text)\n",
    "    #-PRON- happens when the word is a pronoun. In that case, return the word again\n",
    "    text = ' '.join([word.lemma_ if (word.lemma_ != '-PRON-' and word.lemma not in list_of_stopwords) else word.text for word in text])\n",
    "    return text\n",
    "\n",
    "def remove_special_chars(text):\n",
    "    \"\"\"\n",
    "    if the character is not a space and regular alphabet,\n",
    "    the character will be removed. However \"_\" will not be\n",
    "    removed\n",
    "    \"\"\"\n",
    "    char = r'[^A-z0-9\\s]'\n",
    "    possessive_char = r'\\ss\\s'\n",
    "    white_space=r'  '\n",
    "    text = re.sub(char, '', text) #removing\n",
    "    text = re.sub(possessive_char,'',text)\n",
    "    text = re.sub(white_space,' ',text)\n",
    "    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n",
    "    text = re.sub('<.*?>+', '', text)\n",
    "    text = re.sub('\\n', '', text)\n",
    "    text = re.sub('\\w*\\d\\w*', '', text)\n",
    "    return text\n",
    "\n",
    "def removeNumbers(text):\n",
    "    \"\"\" Removes integers \"\"\"\n",
    "    text = ''.join([i for i in text if not i.isdigit()])         \n",
    "    return text\n",
    "\n",
    "def replaceMultiExclamationMark(text):\n",
    "    \"\"\" Replaces repetitions of exlamation marks \"\"\"\n",
    "    text = re.sub(r\"(\\!)\\1+\", ' multiExclamation ', text)\n",
    "    return text\n",
    "\n",
    "def replaceMultiQuestionMark(text):\n",
    "    \"\"\" Replaces repetitions of question marks \"\"\"\n",
    "    text = re.sub(r\"(\\?)\\1+\", ' multiQuestion ', text)\n",
    "    return text\n",
    "\n",
    "def replaceMultiStopMark(text):\n",
    "    \"\"\" Replaces repetitions of stop marks \"\"\"\n",
    "    text = re.sub(r\"(\\.)\\1+\", ' multiStop ', text)\n",
    "    return text\n",
    "\n",
    "def replaceContraction(text):\n",
    "        \n",
    "    contraction_patterns = [ (r'won\\'t', 'will not'), (r'can\\'t', 'cannot'), (r'i\\'m', 'i am'), (r'ain\\'t', 'is not'), (r'(\\w+)\\'ll', '\\g<1> will'), (r'(\\w+)n\\'t', '\\g<1> not'),\n",
    "                     (r'(\\w+)\\'ve', '\\g<1> have'), (r'(\\w+)\\'s', '\\g<1> is'), (r'(\\w+)\\'re', '\\g<1> are'), (r'(\\w+)\\'d', '\\g<1> would'), (r'&', 'and'), (r'dammit', 'damn it'), (r'dont', 'do not'), (r'wont', 'will not') ]\n",
    "    patterns = [(re.compile(regex), repl) for (regex, repl) in contraction_patterns]\n",
    "    for (pattern, repl) in patterns:\n",
    "        (text, count) = re.subn(pattern, repl, text)\n",
    "    return text\n",
    "## WARNING: SLOW PROCESS\n",
    "ser = ser.apply(lambda x: lemmatize_text(x))\n",
    "ser = ser.apply(lambda x: remove_special_chars(x))\n",
    "ser = ser.apply(lambda x: removeNumbers(x))\n",
    "ser = ser.apply(lambda x: replaceMultiExclamationMark(x))\n",
    "ser = ser.apply(lambda x: replaceMultiQuestionMark(x))\n",
    "ser = ser.apply(lambda x: replaceMultiStopMark(x))\n",
    "ser = ser.apply(lambda x: replaceContraction(x))\n",
    "\n",
    "print(ser.describe())\n",
    "ser.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e28334d",
   "metadata": {},
   "source": [
    "#### The lemmatized and cleaned text is saved to csv because it takes a long time to process again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ac024a5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ser.to_csv('lemmatized_text.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fccbb7d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}