ads_anaylsis_CT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "05e4e1c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pandas\n",
      "  Downloading pandas-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl (11.1 MB)\n",
      "     |████████████████████████████████| 11.1 MB 5.8 MB/s            \n",
      "\u001b[?25hCollecting numpy>=1.18.5\n",
      "  Using cached numpy-1.22.3-cp39-cp39-macosx_10_14_x86_64.whl (17.6 MB)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from pandas) (2022.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/six/1.16.0_2/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",
      "Installing collected packages: numpy, pandas\n",
      "Successfully installed numpy-1.22.3 pandas-1.4.2\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b059536f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting spacy\n",
      "  Downloading spacy-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl (6.5 MB)\n",
      "     |████████████████████████████████| 6.5 MB 5.0 MB/s            \n",
      "\u001b[?25hCollecting murmurhash<1.1.0,>=0.28.0\n",
      "  Downloading murmurhash-1.0.7-cp39-cp39-macosx_10_9_x86_64.whl (18 kB)\n",
      "Collecting srsly<3.0.0,>=2.4.3\n",
      "  Downloading srsly-2.4.3-cp39-cp39-macosx_10_9_x86_64.whl (457 kB)\n",
      "     |████████████████████████████████| 457 kB 4.6 MB/s            \n",
      "\u001b[?25hCollecting pathy>=0.3.5\n",
      "  Using cached pathy-0.6.1-py3-none-any.whl (42 kB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from spacy) (2.27.1)\n",
      "Collecting blis<0.8.0,>=0.4.0\n",
      "  Using cached blis-0.7.7-cp39-cp39-macosx_10_9_x86_64.whl (5.8 MB)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from spacy) (21.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from spacy) (60.5.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from spacy) (1.22.3)\n",
      "Collecting preshed<3.1.0,>=3.0.2\n",
      "  Using cached preshed-3.0.6-cp39-cp39-macosx_10_9_x86_64.whl (106 kB)\n",
      "Requirement already satisfied: jinja2 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from spacy) (3.0.3)\n",
      "Collecting typer<0.5.0,>=0.3.0\n",
      "  Using cached typer-0.4.1-py3-none-any.whl (27 kB)\n",
      "Collecting pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4\n",
      "  Using cached pydantic-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl (2.7 MB)\n",
      "Collecting spacy-legacy<3.1.0,>=3.0.9\n",
      "  Using cached spacy_legacy-3.0.9-py2.py3-none-any.whl (20 kB)\n",
      "Collecting langcodes<4.0.0,>=3.2.0\n",
      "  Using cached langcodes-3.3.0-py3-none-any.whl (181 kB)\n",
      "Collecting spacy-loggers<2.0.0,>=1.0.0\n",
      "  Using cached spacy_loggers-1.0.2-py3-none-any.whl (7.2 kB)\n",
      "Collecting wasabi<1.1.0,>=0.9.1\n",
      "  Using cached wasabi-0.9.1-py3-none-any.whl (26 kB)\n",
      "Collecting tqdm<5.0.0,>=4.38.0\n",
      "  Using cached tqdm-4.64.0-py2.py3-none-any.whl (78 kB)\n",
      "Collecting cymem<2.1.0,>=2.0.2\n",
      "  Using cached cymem-2.0.6-cp39-cp39-macosx_10_9_x86_64.whl (32 kB)\n",
      "Collecting catalogue<2.1.0,>=2.0.6\n",
      "  Using cached catalogue-2.0.7-py3-none-any.whl (17 kB)\n",
      "Collecting thinc<8.1.0,>=8.0.14\n",
      "  Using cached thinc-8.0.15-cp39-cp39-macosx_10_9_x86_64.whl (639 kB)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from packaging>=20.0->spacy) (3.0.7)\n",
      "Collecting smart-open<6.0.0,>=5.0.0\n",
      "  Using cached smart_open-5.2.1-py3-none-any.whl (58 kB)\n",
      "Collecting typing-extensions>=3.7.4.3\n",
      "  Downloading typing_extensions-4.2.0-py3-none-any.whl (24 kB)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2021.10.8)\n",
      "Requirement already satisfied: charset_normalizer~=2.0.0 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.3)\n",
      "Collecting click<9.0.0,>=7.1.1\n",
      "  Downloading click-8.1.3-py3-none-any.whl (96 kB)\n",
      "     |████████████████████████████████| 96 kB 5.8 MB/s             \n",
      "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/Cellar/jupyterlab/3.3.2/libexec/lib/python3.9/site-packages (from jinja2->spacy) (2.1.1)\n",
      "Installing collected packages: typing-extensions, murmurhash, cymem, click, catalogue, wasabi, typer, srsly, smart-open, pydantic, preshed, blis, tqdm, thinc, spacy-loggers, spacy-legacy, pathy, langcodes, spacy\n",
      "Successfully installed blis-0.7.7 catalogue-2.0.7 click-8.1.3 cymem-2.0.6 langcodes-3.3.0 murmurhash-1.0.7 pathy-0.6.1 preshed-3.0.6 pydantic-1.8.2 smart-open-5.2.1 spacy-3.3.0 spacy-legacy-3.0.9 spacy-loggers-1.0.2 srsly-2.4.3 thinc-8.0.15 tqdm-4.64.0 typer-0.4.1 typing-extensions-4.2.0 wasabi-0.9.1\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f2dce9b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import requests\n",
    "import re\n",
    "import json\n",
    "import spacy\n",
    "from spacy.matcher import PhraseMatcher\n",
    "from ipynb.fs.full.helper import get_text_from_csv,read_fb,clean_text,check_null,find_words,find_topic,get_word_lable,count_topic,count_word\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc5645e6",
   "metadata": {},
   "source": [
    "# GOOGLE + FACEBOOK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a44f169",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"It's time to focus on the hard-working famili...</td>\n",
       "      <td>GEORGE LOGAN FOR CONGRESS</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 20...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 20...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"Help Herschel Walker reach his critical end o...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 20...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\"We must prioritize lowering barriers between ...</td>\n",
       "      <td>PhRMA</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>\"Gov. Lamont is touting a proposal that would ...</td>\n",
       "      <td>PhRMA</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>\"There’s no greater priority than defending de...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>\"Leora Is Running For Senate to Retire Dick Bl...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>\"The January 6 insurrection kicked off a conce...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 20...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>\"Leora Is Running For Senate to Retire Dick Bl...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>\"Take the politics out of the treasury and inv...</td>\n",
       "      <td>Arora For CT</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 20...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>\"Take the politics out of the treasury and inv...</td>\n",
       "      <td>Arora For CT</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>\"Tell Governor Lamont CT working families can'...</td>\n",
       "      <td>BOB FOR GOVERNOR INC</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>\"Tell your state legislators: Patients, not mi...</td>\n",
       "      <td>PhRMA</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>\"If we want action on the problems facing Amer...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>\"Leora Levy is the Best Conservative to Take O...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>\"States are passing laws to silence voters. Fe...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>\"The filibuster stands in the way. We need to ...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>\"Dick Blumenthal and Joe Biden’s policies are ...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>\"Don’t be Left Behind. Support a True Conserva...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>\"McConnell enriches his wealthy patrons and us...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>\"Leora Levy has lived the American dream. She ...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>\"Leora is the Conservative Who Can Win in Nove...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>\"Ned Lamont is working for us, investing in go...</td>\n",
       "      <td>Ned for CT</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>\"Don’t be Left Behind. Support a True Conserva...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>\"The Connecticut dream is worth fighting for! ...</td>\n",
       "      <td>THEMIS 2022 INC</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>\"The filibuster is used to block protections f...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>\"The filibuster is used to block protections f...</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>\"Leora is a Jewish Hispanic woman who will say...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>\"Leora knows what happens when we allow Mobs t...</td>\n",
       "      <td>Leora Levy for U.S. Senate, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quart...</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  \\\n",
       "0   \"Our Democrat opponent outraised us last quart...   \n",
       "1   \"Our Democrat opponent outraised us last quart...   \n",
       "2   \"It's time to focus on the hard-working famili...   \n",
       "3   \"Georgia will decide the Senate Majority in 20...   \n",
       "4   \"Georgia will decide the Senate Majority in 20...   \n",
       "5   \"Our Democrat opponent outraised us last quart...   \n",
       "6   \"Help Herschel Walker reach his critical end o...   \n",
       "7   \"Georgia will decide the Senate Majority in 20...   \n",
       "8   \"We must prioritize lowering barriers between ...   \n",
       "9   \"Gov. Lamont is touting a proposal that would ...   \n",
       "10  \"There’s no greater priority than defending de...   \n",
       "11  \"Leora Is Running For Senate to Retire Dick Bl...   \n",
       "12  \"The January 6 insurrection kicked off a conce...   \n",
       "13  \"Georgia will decide the Senate Majority in 20...   \n",
       "14  \"Leora Is Running For Senate to Retire Dick Bl...   \n",
       "15  \"Our Democrat opponent outraised us last quart...   \n",
       "16  \"Our Democrat opponent outraised us last quart...   \n",
       "17  \"Our Democrat opponent outraised us last quart...   \n",
       "18  \"Our Democrat opponent outraised us last quart...   \n",
       "19  \"Take the politics out of the treasury and inv...   \n",
       "20  \"Our Democrat opponent outraised us last quart...   \n",
       "21  \"Georgia will decide the Senate Majority in 20...   \n",
       "22  \"Our Democrat opponent outraised us last quart...   \n",
       "23  \"Take the politics out of the treasury and inv...   \n",
       "24  \"Our Democrat opponent outraised us last quart...   \n",
       "25  \"Our Democrat opponent outraised us last quart...   \n",
       "26  \"Tell Governor Lamont CT working families can'...   \n",
       "27  \"Tell your state legislators: Patients, not mi...   \n",
       "28  \"If we want action on the problems facing Amer...   \n",
       "29  \"Leora Levy is the Best Conservative to Take O...   \n",
       "30  \"States are passing laws to silence voters. Fe...   \n",
       "31  \"The filibuster stands in the way. We need to ...   \n",
       "32  \"Dick Blumenthal and Joe Biden’s policies are ...   \n",
       "33  \"Don’t be Left Behind. Support a True Conserva...   \n",
       "34  \"Our Democrat opponent outraised us last quart...   \n",
       "35  \"Our Democrat opponent outraised us last quart...   \n",
       "36  \"Our Democrat opponent outraised us last quart...   \n",
       "37  \"Our Democrat opponent outraised us last quart...   \n",
       "38  \"Our Democrat opponent outraised us last quart...   \n",
       "39  \"McConnell enriches his wealthy patrons and us...   \n",
       "40  \"Leora Levy has lived the American dream. She ...   \n",
       "41  \"Our Democrat opponent outraised us last quart...   \n",
       "42  \"Leora is the Conservative Who Can Win in Nove...   \n",
       "43  \"Ned Lamont is working for us, investing in go...   \n",
       "44  \"Our Democrat opponent outraised us last quart...   \n",
       "45  \"Our Democrat opponent outraised us last quart...   \n",
       "46  \"Don’t be Left Behind. Support a True Conserva...   \n",
       "47  \"The Connecticut dream is worth fighting for! ...   \n",
       "48  \"The filibuster is used to block protections f...   \n",
       "49  \"The filibuster is used to block protections f...   \n",
       "50  \"Our Democrat opponent outraised us last quart...   \n",
       "51  \"Leora is a Jewish Hispanic woman who will say...   \n",
       "52  \"Leora knows what happens when we allow Mobs t...   \n",
       "53  \"Our Democrat opponent outraised us last quart...   \n",
       "\n",
       "                     advertiser_name platform  \n",
       "0                Team Herschel, Inc.   google  \n",
       "1                Team Herschel, Inc.   google  \n",
       "2          GEORGE LOGAN FOR CONGRESS   google  \n",
       "3                Team Herschel, Inc.   google  \n",
       "4                Team Herschel, Inc.   google  \n",
       "5                Team Herschel, Inc.   google  \n",
       "6                Team Herschel, Inc.   google  \n",
       "7                Team Herschel, Inc.   google  \n",
       "8                              PhRMA   google  \n",
       "9                              PhRMA   google  \n",
       "10              Democracy Initiative   google  \n",
       "11  Leora Levy for U.S. Senate, Inc.   google  \n",
       "12              Democracy Initiative   google  \n",
       "13               Team Herschel, Inc.   google  \n",
       "14  Leora Levy for U.S. Senate, Inc.   google  \n",
       "15               Team Herschel, Inc.   google  \n",
       "16               Team Herschel, Inc.   google  \n",
       "17               Team Herschel, Inc.   google  \n",
       "18               Team Herschel, Inc.   google  \n",
       "19                      Arora For CT   google  \n",
       "20               Team Herschel, Inc.   google  \n",
       "21               Team Herschel, Inc.   google  \n",
       "22               Team Herschel, Inc.   google  \n",
       "23                      Arora For CT   google  \n",
       "24               Team Herschel, Inc.   google  \n",
       "25               Team Herschel, Inc.   google  \n",
       "26              BOB FOR GOVERNOR INC   google  \n",
       "27                             PhRMA   google  \n",
       "28              Democracy Initiative   google  \n",
       "29  Leora Levy for U.S. Senate, Inc.   google  \n",
       "30              Democracy Initiative   google  \n",
       "31              Democracy Initiative   google  \n",
       "32  Leora Levy for U.S. Senate, Inc.   google  \n",
       "33  Leora Levy for U.S. Senate, Inc.   google  \n",
       "34               Team Herschel, Inc.   google  \n",
       "35               Team Herschel, Inc.   google  \n",
       "36               Team Herschel, Inc.   google  \n",
       "37               Team Herschel, Inc.   google  \n",
       "38               Team Herschel, Inc.   google  \n",
       "39              Democracy Initiative   google  \n",
       "40  Leora Levy for U.S. Senate, Inc.   google  \n",
       "41               Team Herschel, Inc.   google  \n",
       "42  Leora Levy for U.S. Senate, Inc.   google  \n",
       "43                        Ned for CT   google  \n",
       "44               Team Herschel, Inc.   google  \n",
       "45               Team Herschel, Inc.   google  \n",
       "46  Leora Levy for U.S. Senate, Inc.   google  \n",
       "47                   THEMIS 2022 INC   google  \n",
       "48              Democracy Initiative   google  \n",
       "49              Democracy Initiative   google  \n",
       "50               Team Herschel, Inc.   google  \n",
       "51  Leora Levy for U.S. Senate, Inc.   google  \n",
       "52  Leora Levy for U.S. Senate, Inc.   google  \n",
       "53               Team Herschel, Inc.   google  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "google_df = get_text_from_csv('GoogleAds/CT.csv')\n",
    "google_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f211b48",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "defe60bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Please join CEA's Red for Ed Day of Action this Wednesday! Come to Hartford to meet with your legislators, or if you can't make it, take just a few minutes to call or email.</td>\n",
       "      <td>CONNECTICUT EDUCATION ASSOCIATION</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Thinking about attending a Connecticut State Community College? Get all the information and support you need during our virtual showcases April 28-30th. Hear about community college and program options throughout the state, transfer opportunities, financial aid, careers and more all from your couch! Live chat will be available. Register today by visiting the link in our bio.</td>\n",
       "      <td>odonnell company</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Governor Lamont is protecting abortion rights in Connecticut. The governor is set to sign a major piece of legislation protecting abortion ahead of an upcoming Supreme Court ruling. Sign up now to learn more about what the Governor is doing for Connecticut &gt;&gt;</td>\n",
       "      <td>NED FOR CT</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Hey Connecticut, have you heard? Governor Lamont has signed a bill suspending the gas tax. This means real relief for working families across the state. Add your name now to get more updates on what Gov. Lamont is doing for Connecticut &gt;&gt;</td>\n",
       "      <td>NED FOR CT</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cost increases are hitting families in Connecticut. That’s why Governor Lamont and the Legislature have suspended the gas tax. It’ll cost you less to fill up at the pump. That’s real relief for Connecticut families. Add your name and find out more about how the Governor is working for you.</td>\n",
       "      <td>NED FOR CT</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1280</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1281</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1282</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1284</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1285 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                           text  \\\n",
       "0                                                                                                                                                                                                                 Please join CEA's Red for Ed Day of Action this Wednesday! Come to Hartford to meet with your legislators, or if you can't make it, take just a few minutes to call or email.   \n",
       "1     Thinking about attending a Connecticut State Community College? Get all the information and support you need during our virtual showcases April 28-30th. Hear about community college and program options throughout the state, transfer opportunities, financial aid, careers and more all from your couch! Live chat will be available. Register today by visiting the link in our bio.   \n",
       "2                                                                                                                           Governor Lamont is protecting abortion rights in Connecticut. The governor is set to sign a major piece of legislation protecting abortion ahead of an upcoming Supreme Court ruling. Sign up now to learn more about what the Governor is doing for Connecticut >>   \n",
       "3                                                                                                                                                Hey Connecticut, have you heard? Governor Lamont has signed a bill suspending the gas tax. This means real relief for working families across the state. Add your name now to get more updates on what Gov. Lamont is doing for Connecticut >>   \n",
       "4                                                                                            Cost increases are hitting families in Connecticut. That’s why Governor Lamont and the Legislature have suspended the gas tax. It’ll cost you less to fill up at the pump. That’s real relief for Connecticut families. Add your name and find out more about how the Governor is working for you.   \n",
       "...                                                                                                                                                                                                                                                                                                                                                                                         ...   \n",
       "1280                                                                                                                                                             Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1281                                                                                                                                                             Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1282                                                                                                                                                             Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1283                                                                                                                                                             Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1284                                                                                                                                                             Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "\n",
       "                        advertiser_name  platform  \n",
       "0     CONNECTICUT EDUCATION ASSOCIATION  facebook  \n",
       "1                      odonnell company  facebook  \n",
       "2                            NED FOR CT  facebook  \n",
       "3                            NED FOR CT  facebook  \n",
       "4                            NED FOR CT  facebook  \n",
       "...                                 ...       ...  \n",
       "1280                                NaN  facebook  \n",
       "1281                                NaN  facebook  \n",
       "1282                                NaN  facebook  \n",
       "1283                                NaN  facebook  \n",
       "1284                                NaN  facebook  \n",
       "\n",
       "[1285 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebook_df = read_fb('FacebookAds/Connecticut.csv')\n",
    "facebook_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e59b9dec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quarter, so we need your help immediately.\",\"The Georgia polls are a dead heat, so please help us close the fundraising gap today!</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Our Democrat opponent outraised us last quarter, so we need your help immediately.\",\"Georgia will decide the Senate Majority in 2022, so help us close the fundraising gap!</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"It's time to focus on the hard-working families in Connecticut and end the D.C. power grab\",\"George Logan is the Republican leadership Washington and Connecticut need. Donate today!</td>\n",
       "      <td>GEORGE LOGAN FOR CONGRESS</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 2022, so help us close the fundraising gap!\",\"Our Democrat opponent outraised us last quarter, so we need your help immediately.</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"Georgia will decide the Senate Majority in 2022, so help us close the fundraising gap!\",\"Our Democrat opponent outraised us last quarter, so we need your help immediately.</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1280</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1281</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1282</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1284</th>\n",
       "      <td>Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1339 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                text  \\\n",
       "0                                                        \"Our Democrat opponent outraised us last quarter, so we need your help immediately.\",\"The Georgia polls are a dead heat, so please help us close the fundraising gap today!   \n",
       "1                                                       \"Our Democrat opponent outraised us last quarter, so we need your help immediately.\",\"Georgia will decide the Senate Majority in 2022, so help us close the fundraising gap!   \n",
       "2                                             \"It's time to focus on the hard-working families in Connecticut and end the D.C. power grab\",\"George Logan is the Republican leadership Washington and Connecticut need. Donate today!   \n",
       "3                                                       \"Georgia will decide the Senate Majority in 2022, so help us close the fundraising gap!\",\"Our Democrat opponent outraised us last quarter, so we need your help immediately.   \n",
       "4                                                       \"Georgia will decide the Senate Majority in 2022, so help us close the fundraising gap!\",\"Our Democrat opponent outraised us last quarter, so we need your help immediately.   \n",
       "...                                                                                                                                                                                                                              ...   \n",
       "1280  Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1281  Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1282  Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1283  Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "1284  Connecticut Dairy farmers understand the importance of thoughtful, yet vigorous work to increase sustainability on the farm. Watch and learn how two local farms are tackling climate change with soil management practices. 🌱   \n",
       "\n",
       "                advertiser_name  platform  \n",
       "0           Team Herschel, Inc.    google  \n",
       "1           Team Herschel, Inc.    google  \n",
       "2     GEORGE LOGAN FOR CONGRESS    google  \n",
       "3           Team Herschel, Inc.    google  \n",
       "4           Team Herschel, Inc.    google  \n",
       "...                         ...       ...  \n",
       "1280                        NaN  facebook  \n",
       "1281                        NaN  facebook  \n",
       "1282                        NaN  facebook  \n",
       "1283                        NaN  facebook  \n",
       "1284                        NaN  facebook  \n",
       "\n",
       "[1339 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cncat google_df and facebook_df by 'text'\n",
    "df_text = pd.concat([google_df,facebook_df])\n",
    "df_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f69eed0e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "54cf701a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 234 entries, 0 to 233\n",
      "Data columns (total 3 columns):\n",
      " #   Column           Non-Null Count  Dtype \n",
      "---  ------           --------------  ----- \n",
      " 0   text             234 non-null    object\n",
      " 1   advertiser_name  226 non-null    object\n",
      " 2   platform         234 non-null    object\n",
      "dtypes: object(3)\n",
      "memory usage: 5.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df_text = clean_text(df_text)\n",
    "df_text.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fa3335fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our democrat opponent outraised us last quarter so we need your help immediately the georgia polls are a dead heat so please help us close the fundraising gap today</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>our democrat opponent outraised us last quarter so we need your help immediately georgia will decide the senate majority in 2022 so help us close the fundraising gap</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>it s time to focus on the hard working families in connecticut and end the d c power grab george logan is the republican leadership washington and connecticut need donate today</td>\n",
       "      <td>GEORGE LOGAN FOR CONGRESS</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>help herschel walker reach his critical end of quarter fundraising goal donate to the official website of herschel walker for united states senate today</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>georgia will decide the senate majority in 2022 so help us close the fundraising gap our democrat opponent outraised us last quarter so we need your help immediately</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                               text  \\\n",
       "0              our democrat opponent outraised us last quarter so we need your help immediately the georgia polls are a dead heat so please help us close the fundraising gap today   \n",
       "1             our democrat opponent outraised us last quarter so we need your help immediately georgia will decide the senate majority in 2022 so help us close the fundraising gap   \n",
       "2  it s time to focus on the hard working families in connecticut and end the d c power grab george logan is the republican leadership washington and connecticut need donate today   \n",
       "3                          help herschel walker reach his critical end of quarter fundraising goal donate to the official website of herschel walker for united states senate today   \n",
       "4             georgia will decide the senate majority in 2022 so help us close the fundraising gap our democrat opponent outraised us last quarter so we need your help immediately   \n",
       "\n",
       "             advertiser_name platform  \n",
       "0        Team Herschel, Inc.   google  \n",
       "1        Team Herschel, Inc.   google  \n",
       "2  GEORGE LOGAN FOR CONGRESS   google  \n",
       "3        Team Herschel, Inc.   google  \n",
       "4        Team Herschel, Inc.   google  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_text.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "abfbce47",
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[E050] Can't find model '/usr/local/Cellar/jupyterlab/3.2.9/libexec/lib/python3.9/site-packages/en_core_web_lg/en_core_web_lg-3.2.0/'. It doesn't seem to be a Python package or a valid path to a data directory.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Input \u001b[0;32mIn [11]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df_text \u001b[38;5;241m=\u001b[39m \u001b[43mget_word_lable\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf_text\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlexicon.json\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m df_text\n",
      "File \u001b[0;32m~/git/ml-political-ads/helper.ipynb:102\u001b[0m, in \u001b[0;36mget_word_lable\u001b[0;34m(df_text, lexicon)\u001b[0m\n\u001b[1;32m      1\u001b[0m {\n\u001b[1;32m      2\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcells\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m      3\u001b[0m   {\n\u001b[1;32m      4\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      5\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m4\u001b[39m,\n\u001b[1;32m      6\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m88acc7d5\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      7\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m      8\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m      9\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom bs4 import BeautifulSoup\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport pandas as pd\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     12\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport requests\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     13\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport re\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     14\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport json\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport spacy\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom spacy.matcher import PhraseMatcher\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     17\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport numpy as np\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     18\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom youtube_transcript_api import YouTubeTranscriptApi\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     19\u001b[0m    ]\n\u001b[1;32m     20\u001b[0m   },\n\u001b[1;32m     21\u001b[0m   {\n\u001b[1;32m     22\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     23\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m5\u001b[39m,\n\u001b[1;32m     24\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m4c72fcd3\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     25\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     26\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     27\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     28\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_text_from_csv(fn): # get ad\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms text from csv file, return a dataframe with text, ad_id,ad_url, and report_url\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     29\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     30\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mText\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].reset_index(drop = True) # only get text ads\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_url = df[[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) # get the urls\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    urls = df_url[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].to_list() # create a list so we could get the report_urls\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     33\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    report_urls = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     34\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for url in urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        entity_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     36\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        creative_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     37\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://transparencyreport.google.com/transparencyreport/api/v3/politicalads/creatives/details?entity_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&creative_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&hl=en\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.format(entity_id,creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_urls.append(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     39\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    ad_text = [] # get texts from reports\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     40\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for report_url in report_urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     41\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        response = requests.get(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        text = response.text.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[0].split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        ad_text.append(text)  \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     44\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text = pd.DataFrame(ad_text).rename(columns = \u001b[39m\u001b[38;5;124m{\u001b[39m\u001b[38;5;124m0:\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m})\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_new = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mText\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m][[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     46\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.concat([df_text,df_new],axis=1)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplatform\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgoogle\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     48\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df  \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     49\u001b[0m    ]\n\u001b[1;32m     50\u001b[0m   },\n\u001b[1;32m     51\u001b[0m   {\n\u001b[1;32m     52\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     53\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m6\u001b[39m,\n\u001b[1;32m     54\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m5b928802\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     55\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     56\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     57\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     58\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# check the dataset \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     59\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef check_null(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].isnull().sum():\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        print (df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].isnull().sum())\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     62\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        df_text = df_text[df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].notnull()]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     63\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        df_text.reset_index(drop=True,inplace=True)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     64\u001b[0m    ]\n\u001b[1;32m     65\u001b[0m   },\n\u001b[1;32m     66\u001b[0m   {\n\u001b[1;32m     67\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     68\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m7\u001b[39m,\n\u001b[1;32m     69\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0daaeae0\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     70\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     71\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     72\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     73\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# function that filter the urls and symbols in the text \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     74\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef clean_text(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     75\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    def filter_text(x):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     76\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp[s]?://\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mS+\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     77\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(url,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     78\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m[^\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mw\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124ms]\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x) # filter symbols\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     79\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124ms+\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     80\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        ls=[w.lower() for w in x.split()] \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     81\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(ls)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     82\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].astype(str).apply(lambda x: filter_text(x))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     83\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].replace(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/u0026\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text.drop_duplicates(subset = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,keep = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     85\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text.reset_index(drop = True,inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_text\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     87\u001b[0m    ]\n\u001b[1;32m     88\u001b[0m   },\n\u001b[1;32m     89\u001b[0m   {\n\u001b[1;32m     90\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     91\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m8\u001b[39m,\n\u001b[1;32m     92\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma3c8ef8c\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     93\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     94\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     95\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     96\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# the function that find the lexicon words in the text\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef find_words(x,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     98\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    topics= lexicon.keys()  \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     99\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    nlp = spacy.load(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m/usr/local/Cellar/jupyterlab/3.2.9/libexec/lib/python3.9/site-packages/en_core_web_lg/en_core_web_lg-3.2.0/\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    100\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    doc = nlp(x) # nlp() is spaCy 2.2 English language model \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    101\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    words= []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m--> 102\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for t in topics:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matcher = PhraseMatcher(nlp.vocab)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        terms= lexicon[t]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    105\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        patterns = [nlp.make_doc(text) for text in terms]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    106\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matcher.add(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124mTerminologyList\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m, patterns) # spaCy2.2 phrase matcher\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    107\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matches = matcher(doc)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    108\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        for match_id, start,end in matches:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    109\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            span = doc[start:end]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    110\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            words.append(span.text)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    111\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if words:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    112\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        words = list(set(words))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    113\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(words)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    else:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    115\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    116\u001b[0m    ]\n\u001b[1;32m    117\u001b[0m   },\n\u001b[1;32m    118\u001b[0m   {\n\u001b[1;32m    119\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    120\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m9\u001b[39m,\n\u001b[1;32m    121\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mff10e181\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    122\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    123\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    124\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# tagging the topic in each message\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef find_topic(x,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    127\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    topics= lexicon.keys()    \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    128\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if x==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    129\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m    \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    130\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if x != \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    131\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        words = x.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    132\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        labels = []        \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    133\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        for t in topics:            \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    134\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            terms = lexicon[t]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    135\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            if set(words)&set(terms):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    136\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                labels.append(t)                \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    137\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                #l = sorted(labels)        \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    138\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return  \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(sorted(labels))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    139\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    140\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        #return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(labels)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    141\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    142\u001b[0m    ]\n\u001b[1;32m    143\u001b[0m   },\n\u001b[1;32m    144\u001b[0m   {\n\u001b[1;32m    145\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    146\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m10\u001b[39m,\n\u001b[1;32m    147\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdc891547\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    148\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    149\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    150\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    151\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_word_lable(df_text,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    152\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    with open (lexicon) as f:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    153\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        dic = json.load(f)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].astype(str).apply(lambda x: find_words(x,dic))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    155\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].apply(lambda x: find_topic(x,dic))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    156\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].apply(lambda x: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m if x==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m else x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    157\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_text\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    158\u001b[0m    ]\n\u001b[1;32m    159\u001b[0m   },\n\u001b[1;32m    160\u001b[0m   {\n\u001b[1;32m    161\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    162\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m11\u001b[39m,\n\u001b[1;32m    163\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m7f5e156b\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    164\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    165\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    166\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    167\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef count_topic(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    168\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].value_counts().rename_axis(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtopics\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(name=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    169\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_tag.assign(single_topic=df_tag[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtopics\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)).explode(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(drop = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    170\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_tag.groupby(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).sum().reset_index().sort_values(by = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, ascending = False)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    171\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag.reset_index(drop = True, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    172\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_tag\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    173\u001b[0m    ]\n\u001b[1;32m    174\u001b[0m   },\n\u001b[1;32m    175\u001b[0m   {\n\u001b[1;32m    176\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    177\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m12\u001b[39m,\n\u001b[1;32m    178\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m9be8a97d\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    179\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    180\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    181\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    182\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef count_word(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    183\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].value_counts().rename_axis(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(name=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    184\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_words.assign(single_word=df_words[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)).explode(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_word\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(drop = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    185\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_words.groupby(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_word\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).sum().reset_index().sort_values(by = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, ascending = False)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    186\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words.reset_index(drop = True, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    187\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_words\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    188\u001b[0m    ]\n\u001b[1;32m    189\u001b[0m   },\n\u001b[1;32m    190\u001b[0m   {\n\u001b[1;32m    191\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    192\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m13\u001b[39m,\n\u001b[1;32m    193\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbe9a9f65\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    194\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    195\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    196\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    197\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef read_fb(fn):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    198\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    199\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df.drop(columns = [\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnnamed: 0\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m],inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    200\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df.rename(columns = \u001b[39m\u001b[38;5;124m{\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbyline\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m},inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    201\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    pd.set_option(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdisplay.max_colwidth\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, None)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    202\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplatform\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfacebook\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    203\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return facebook_df\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    204\u001b[0m    ]\n\u001b[1;32m    205\u001b[0m   },\n\u001b[1;32m    206\u001b[0m   {\n\u001b[1;32m    207\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    208\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m14\u001b[39m,\n\u001b[1;32m    209\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m04528f75\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    210\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    211\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    212\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    213\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# go through the csv file, get youtube video\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms ids from all video ads. Add new column \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    214\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# before running this function, make sure the csv file includes ad_url\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    215\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_youtube_id(fn): \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    216\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    217\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mVideo\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].reset_index(drop = True) # only get video ads\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    218\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_video = df[[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mimpressions\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mspend_usd\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    219\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    urls = df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].to_list() # \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    220\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    youtube_ids = [] # get youtube id\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    221\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for url in urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    222\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        entity_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    223\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        creative_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    224\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://transparencyreport.google.com/transparencyreport/api/v3/politicalads/creatives/details?entity_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&creative_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&hl=en\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.format(entity_id,creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    225\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        response = requests.get(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    226\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        try:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    227\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            youtube_id = response.text.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    228\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            if len(list(youtube_id)) > 11: # mark ads violating google polices\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                youtube_id = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id not available: this ad violated google ad policies.\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    230\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            #print(youtube_id + \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m + creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    231\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        except IndexError: # mark ads cannot be loaded\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    232\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            youtube_id = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id not available: cannot load the video with this ad_url.\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    233\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            #print(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124mcan\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt load this video, \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m + report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    234\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            pass\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        youtube_ids.append(youtube_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    236\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = youtube_ids\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    237\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    238\u001b[0m    ]\n\u001b[1;32m    239\u001b[0m   },\n\u001b[1;32m    240\u001b[0m   {\n\u001b[1;32m    241\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    242\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m15\u001b[39m,\n\u001b[1;32m    243\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3ac7ad49\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    244\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    245\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    246\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    247\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# check all videos, see which are available and drop duplicates. Add a new column video_available\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef check_video(df_video): \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    249\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  for id in df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    250\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if len(list(id)) == 11:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    251\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video.drop_duplicates(subset = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,keep = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    252\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video.reset_index(drop = True, inplace=True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      yes_video = df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.len() == 11\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    254\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mvideo_available\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = yes_video\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    255\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    256\u001b[0m    ]\n\u001b[1;32m    257\u001b[0m   },\n\u001b[1;32m    258\u001b[0m   {\n\u001b[1;32m    259\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    260\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m16\u001b[39m,\n\u001b[1;32m    261\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbd8f2a85\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    262\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    263\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    264\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    265\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# use youtube_id to get captions. Add a new column youtube_captions\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_captions(df_video):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    267\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  youtube_captions = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    268\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  for youtube_id in df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    try:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    270\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      subs = YouTubeTranscriptApi.get_transcript(youtube_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      alist = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    272\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      for sub in subs:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    273\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        alist.append(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m + sub[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m])\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    274\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      captions = \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    275\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      for item in alist:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    276\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        captions += item\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    277\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    except Exception as e:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    278\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      captions = e   \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    279\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    youtube_captions.append(captions)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    280\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  #print(len(youtube_captions))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    281\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = youtube_captions\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    283\u001b[0m    ]\n\u001b[1;32m    284\u001b[0m   }\n\u001b[1;32m    285\u001b[0m  ],\n\u001b[1;32m    286\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    287\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minterpreter\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    288\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhash\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb2373ebc037b6b887ce6ae759e146cfea08225ef87b9ff98319d1ded9ec6eadc\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    289\u001b[0m   },\n\u001b[1;32m    290\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkernelspec\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    291\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdisplay_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPython 3 (ipykernel)\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    292\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlanguage\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    293\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython3\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    294\u001b[0m   },\n\u001b[1;32m    295\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlanguage_info\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    296\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcodemirror_mode\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    297\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mipython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    298\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m3\u001b[39m\n\u001b[1;32m    299\u001b[0m    },\n\u001b[1;32m    300\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_extension\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.py\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    301\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmimetype\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext/x-python\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    302\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    303\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbconvert_exporter\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    304\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpygments_lexer\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mipython3\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    305\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3.9.12\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    306\u001b[0m   }\n\u001b[1;32m    307\u001b[0m  },\n\u001b[1;32m    308\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbformat\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m4\u001b[39m,\n\u001b[1;32m    309\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbformat_minor\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m    310\u001b[0m }\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/ml-political-ads-gxQjH-2P/lib/python3.10/site-packages/pandas/core/series.py:4433\u001b[0m, in \u001b[0;36mSeries.apply\u001b[0;34m(self, func, convert_dtype, args, **kwargs)\u001b[0m\n\u001b[1;32m   4323\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapply\u001b[39m(\n\u001b[1;32m   4324\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   4325\u001b[0m     func: AggFuncType,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   4328\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m   4329\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataFrame \u001b[38;5;241m|\u001b[39m Series:\n\u001b[1;32m   4330\u001b[0m     \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   4331\u001b[0m \u001b[38;5;124;03m    Invoke function on values of Series.\u001b[39;00m\n\u001b[1;32m   4332\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   4431\u001b[0m \u001b[38;5;124;03m    dtype: float64\u001b[39;00m\n\u001b[1;32m   4432\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 4433\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mSeriesApply\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconvert_dtype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/ml-political-ads-gxQjH-2P/lib/python3.10/site-packages/pandas/core/apply.py:1082\u001b[0m, in \u001b[0;36mSeriesApply.apply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1078\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mf, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m   1079\u001b[0m     \u001b[38;5;66;03m# if we are a string, try to dispatch\u001b[39;00m\n\u001b[1;32m   1080\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapply_str()\n\u001b[0;32m-> 1082\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply_standard\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/ml-political-ads-gxQjH-2P/lib/python3.10/site-packages/pandas/core/apply.py:1137\u001b[0m, in \u001b[0;36mSeriesApply.apply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1131\u001b[0m         values \u001b[38;5;241m=\u001b[39m obj\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mobject\u001b[39m)\u001b[38;5;241m.\u001b[39m_values\n\u001b[1;32m   1132\u001b[0m         \u001b[38;5;66;03m# error: Argument 2 to \"map_infer\" has incompatible type\u001b[39;00m\n\u001b[1;32m   1133\u001b[0m         \u001b[38;5;66;03m# \"Union[Callable[..., Any], str, List[Union[Callable[..., Any], str]],\u001b[39;00m\n\u001b[1;32m   1134\u001b[0m         \u001b[38;5;66;03m# Dict[Hashable, Union[Union[Callable[..., Any], str],\u001b[39;00m\n\u001b[1;32m   1135\u001b[0m         \u001b[38;5;66;03m# List[Union[Callable[..., Any], str]]]]]\"; expected\u001b[39;00m\n\u001b[1;32m   1136\u001b[0m         \u001b[38;5;66;03m# \"Callable[[Any], Any]\"\u001b[39;00m\n\u001b[0;32m-> 1137\u001b[0m         mapped \u001b[38;5;241m=\u001b[39m \u001b[43mlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap_infer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1138\u001b[0m \u001b[43m            \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1139\u001b[0m \u001b[43m            \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[arg-type]\u001b[39;49;00m\n\u001b[1;32m   1140\u001b[0m \u001b[43m            \u001b[49m\u001b[43mconvert\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconvert_dtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1141\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1143\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(mapped) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(mapped[\u001b[38;5;241m0\u001b[39m], ABCSeries):\n\u001b[1;32m   1144\u001b[0m     \u001b[38;5;66;03m# GH#43986 Need to do list(mapped) in order to get treated as nested\u001b[39;00m\n\u001b[1;32m   1145\u001b[0m     \u001b[38;5;66;03m#  See also GH#25959 regarding EA support\u001b[39;00m\n\u001b[1;32m   1146\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m obj\u001b[38;5;241m.\u001b[39m_constructor_expanddim(\u001b[38;5;28mlist\u001b[39m(mapped), index\u001b[38;5;241m=\u001b[39mobj\u001b[38;5;241m.\u001b[39mindex)\n",
      "File \u001b[0;32mpandas/_libs/lib.pyx:2870\u001b[0m, in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/git/ml-political-ads/helper.ipynb:102\u001b[0m, in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m      1\u001b[0m {\n\u001b[1;32m      2\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcells\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m      3\u001b[0m   {\n\u001b[1;32m      4\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      5\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m4\u001b[39m,\n\u001b[1;32m      6\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m88acc7d5\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      7\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m      8\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m      9\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom bs4 import BeautifulSoup\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport pandas as pd\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     12\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport requests\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     13\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport re\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     14\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport json\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport spacy\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom spacy.matcher import PhraseMatcher\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     17\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport numpy as np\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     18\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom youtube_transcript_api import YouTubeTranscriptApi\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     19\u001b[0m    ]\n\u001b[1;32m     20\u001b[0m   },\n\u001b[1;32m     21\u001b[0m   {\n\u001b[1;32m     22\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     23\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m5\u001b[39m,\n\u001b[1;32m     24\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m4c72fcd3\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     25\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     26\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     27\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     28\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_text_from_csv(fn): # get ad\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms text from csv file, return a dataframe with text, ad_id,ad_url, and report_url\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     29\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     30\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mText\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].reset_index(drop = True) # only get text ads\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_url = df[[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) # get the urls\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    urls = df_url[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].to_list() # create a list so we could get the report_urls\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     33\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    report_urls = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     34\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for url in urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        entity_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     36\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        creative_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     37\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://transparencyreport.google.com/transparencyreport/api/v3/politicalads/creatives/details?entity_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&creative_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&hl=en\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.format(entity_id,creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_urls.append(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     39\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    ad_text = [] # get texts from reports\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     40\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for report_url in report_urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     41\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        response = requests.get(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        text = response.text.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[0].split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        ad_text.append(text)  \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     44\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text = pd.DataFrame(ad_text).rename(columns = \u001b[39m\u001b[38;5;124m{\u001b[39m\u001b[38;5;124m0:\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m})\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_new = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mText\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m][[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     46\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.concat([df_text,df_new],axis=1)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplatform\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgoogle\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     48\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df  \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     49\u001b[0m    ]\n\u001b[1;32m     50\u001b[0m   },\n\u001b[1;32m     51\u001b[0m   {\n\u001b[1;32m     52\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     53\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m6\u001b[39m,\n\u001b[1;32m     54\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m5b928802\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     55\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     56\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     57\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     58\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# check the dataset \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     59\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef check_null(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].isnull().sum():\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        print (df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].isnull().sum())\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     62\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        df_text = df_text[df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].notnull()]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     63\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        df_text.reset_index(drop=True,inplace=True)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     64\u001b[0m    ]\n\u001b[1;32m     65\u001b[0m   },\n\u001b[1;32m     66\u001b[0m   {\n\u001b[1;32m     67\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     68\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m7\u001b[39m,\n\u001b[1;32m     69\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0daaeae0\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     70\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     71\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     72\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     73\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# function that filter the urls and symbols in the text \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     74\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef clean_text(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     75\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    def filter_text(x):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     76\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp[s]?://\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mS+\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     77\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(url,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     78\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m[^\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mw\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124ms]\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x) # filter symbols\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     79\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124ms+\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     80\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        ls=[w.lower() for w in x.split()] \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     81\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(ls)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     82\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].astype(str).apply(lambda x: filter_text(x))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     83\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].replace(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/u0026\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text.drop_duplicates(subset = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,keep = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     85\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text.reset_index(drop = True,inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_text\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     87\u001b[0m    ]\n\u001b[1;32m     88\u001b[0m   },\n\u001b[1;32m     89\u001b[0m   {\n\u001b[1;32m     90\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     91\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m8\u001b[39m,\n\u001b[1;32m     92\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma3c8ef8c\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     93\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     94\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     95\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     96\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# the function that find the lexicon words in the text\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef find_words(x,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     98\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    topics= lexicon.keys()  \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     99\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    nlp = spacy.load(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m/usr/local/Cellar/jupyterlab/3.2.9/libexec/lib/python3.9/site-packages/en_core_web_lg/en_core_web_lg-3.2.0/\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    100\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    doc = nlp(x) # nlp() is spaCy 2.2 English language model \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    101\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    words= []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m--> 102\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for t in topics:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matcher = PhraseMatcher(nlp.vocab)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        terms= lexicon[t]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    105\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        patterns = [nlp.make_doc(text) for text in terms]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    106\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matcher.add(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124mTerminologyList\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m, patterns) # spaCy2.2 phrase matcher\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    107\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matches = matcher(doc)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    108\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        for match_id, start,end in matches:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    109\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            span = doc[start:end]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    110\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            words.append(span.text)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    111\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if words:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    112\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        words = list(set(words))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    113\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(words)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    else:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    115\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    116\u001b[0m    ]\n\u001b[1;32m    117\u001b[0m   },\n\u001b[1;32m    118\u001b[0m   {\n\u001b[1;32m    119\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    120\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m9\u001b[39m,\n\u001b[1;32m    121\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mff10e181\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    122\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    123\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    124\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# tagging the topic in each message\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef find_topic(x,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    127\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    topics= lexicon.keys()    \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    128\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if x==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    129\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m    \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    130\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if x != \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    131\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        words = x.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    132\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        labels = []        \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    133\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        for t in topics:            \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    134\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            terms = lexicon[t]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    135\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            if set(words)&set(terms):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    136\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                labels.append(t)                \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    137\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                #l = sorted(labels)        \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    138\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return  \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(sorted(labels))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    139\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    140\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        #return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(labels)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    141\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    142\u001b[0m    ]\n\u001b[1;32m    143\u001b[0m   },\n\u001b[1;32m    144\u001b[0m   {\n\u001b[1;32m    145\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    146\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m10\u001b[39m,\n\u001b[1;32m    147\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdc891547\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    148\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    149\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    150\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    151\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_word_lable(df_text,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    152\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    with open (lexicon) as f:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    153\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        dic = json.load(f)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].astype(str).apply(lambda x: find_words(x,dic))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    155\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].apply(lambda x: find_topic(x,dic))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    156\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].apply(lambda x: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m if x==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m else x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    157\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_text\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    158\u001b[0m    ]\n\u001b[1;32m    159\u001b[0m   },\n\u001b[1;32m    160\u001b[0m   {\n\u001b[1;32m    161\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    162\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m11\u001b[39m,\n\u001b[1;32m    163\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m7f5e156b\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    164\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    165\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    166\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    167\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef count_topic(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    168\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].value_counts().rename_axis(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtopics\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(name=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    169\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_tag.assign(single_topic=df_tag[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtopics\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)).explode(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(drop = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    170\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_tag.groupby(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).sum().reset_index().sort_values(by = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, ascending = False)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    171\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag.reset_index(drop = True, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    172\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_tag\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    173\u001b[0m    ]\n\u001b[1;32m    174\u001b[0m   },\n\u001b[1;32m    175\u001b[0m   {\n\u001b[1;32m    176\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    177\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m12\u001b[39m,\n\u001b[1;32m    178\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m9be8a97d\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    179\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    180\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    181\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    182\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef count_word(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    183\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].value_counts().rename_axis(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(name=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    184\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_words.assign(single_word=df_words[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)).explode(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_word\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(drop = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    185\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_words.groupby(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_word\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).sum().reset_index().sort_values(by = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, ascending = False)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    186\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words.reset_index(drop = True, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    187\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_words\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    188\u001b[0m    ]\n\u001b[1;32m    189\u001b[0m   },\n\u001b[1;32m    190\u001b[0m   {\n\u001b[1;32m    191\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    192\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m13\u001b[39m,\n\u001b[1;32m    193\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbe9a9f65\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    194\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    195\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    196\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    197\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef read_fb(fn):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    198\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    199\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df.drop(columns = [\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnnamed: 0\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m],inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    200\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df.rename(columns = \u001b[39m\u001b[38;5;124m{\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbyline\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m},inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    201\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    pd.set_option(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdisplay.max_colwidth\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, None)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    202\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplatform\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfacebook\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    203\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return facebook_df\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    204\u001b[0m    ]\n\u001b[1;32m    205\u001b[0m   },\n\u001b[1;32m    206\u001b[0m   {\n\u001b[1;32m    207\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    208\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m14\u001b[39m,\n\u001b[1;32m    209\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m04528f75\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    210\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    211\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    212\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    213\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# go through the csv file, get youtube video\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms ids from all video ads. Add new column \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    214\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# before running this function, make sure the csv file includes ad_url\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    215\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_youtube_id(fn): \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    216\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    217\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mVideo\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].reset_index(drop = True) # only get video ads\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    218\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_video = df[[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mimpressions\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mspend_usd\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    219\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    urls = df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].to_list() # \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    220\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    youtube_ids = [] # get youtube id\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    221\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for url in urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    222\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        entity_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    223\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        creative_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    224\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://transparencyreport.google.com/transparencyreport/api/v3/politicalads/creatives/details?entity_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&creative_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&hl=en\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.format(entity_id,creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    225\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        response = requests.get(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    226\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        try:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    227\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            youtube_id = response.text.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    228\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            if len(list(youtube_id)) > 11: # mark ads violating google polices\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                youtube_id = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id not available: this ad violated google ad policies.\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    230\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            #print(youtube_id + \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m + creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    231\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        except IndexError: # mark ads cannot be loaded\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    232\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            youtube_id = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id not available: cannot load the video with this ad_url.\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    233\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            #print(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124mcan\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt load this video, \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m + report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    234\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            pass\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        youtube_ids.append(youtube_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    236\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = youtube_ids\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    237\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    238\u001b[0m    ]\n\u001b[1;32m    239\u001b[0m   },\n\u001b[1;32m    240\u001b[0m   {\n\u001b[1;32m    241\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    242\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m15\u001b[39m,\n\u001b[1;32m    243\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3ac7ad49\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    244\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    245\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    246\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    247\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# check all videos, see which are available and drop duplicates. Add a new column video_available\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef check_video(df_video): \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    249\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  for id in df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    250\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if len(list(id)) == 11:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    251\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video.drop_duplicates(subset = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,keep = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    252\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video.reset_index(drop = True, inplace=True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      yes_video = df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.len() == 11\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    254\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mvideo_available\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = yes_video\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    255\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    256\u001b[0m    ]\n\u001b[1;32m    257\u001b[0m   },\n\u001b[1;32m    258\u001b[0m   {\n\u001b[1;32m    259\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    260\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m16\u001b[39m,\n\u001b[1;32m    261\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbd8f2a85\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    262\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    263\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    264\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    265\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# use youtube_id to get captions. Add a new column youtube_captions\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_captions(df_video):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    267\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  youtube_captions = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    268\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  for youtube_id in df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    try:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    270\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      subs = YouTubeTranscriptApi.get_transcript(youtube_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      alist = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    272\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      for sub in subs:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    273\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        alist.append(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m + sub[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m])\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    274\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      captions = \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    275\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      for item in alist:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    276\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        captions += item\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    277\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    except Exception as e:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    278\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      captions = e   \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    279\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    youtube_captions.append(captions)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    280\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  #print(len(youtube_captions))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    281\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = youtube_captions\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    283\u001b[0m    ]\n\u001b[1;32m    284\u001b[0m   }\n\u001b[1;32m    285\u001b[0m  ],\n\u001b[1;32m    286\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    287\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minterpreter\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    288\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhash\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb2373ebc037b6b887ce6ae759e146cfea08225ef87b9ff98319d1ded9ec6eadc\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    289\u001b[0m   },\n\u001b[1;32m    290\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkernelspec\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    291\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdisplay_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPython 3 (ipykernel)\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    292\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlanguage\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    293\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython3\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    294\u001b[0m   },\n\u001b[1;32m    295\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlanguage_info\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    296\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcodemirror_mode\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    297\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mipython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    298\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m3\u001b[39m\n\u001b[1;32m    299\u001b[0m    },\n\u001b[1;32m    300\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_extension\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.py\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    301\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmimetype\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext/x-python\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    302\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    303\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbconvert_exporter\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    304\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpygments_lexer\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mipython3\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    305\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3.9.12\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    306\u001b[0m   }\n\u001b[1;32m    307\u001b[0m  },\n\u001b[1;32m    308\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbformat\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m4\u001b[39m,\n\u001b[1;32m    309\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbformat_minor\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m    310\u001b[0m }\n",
      "File \u001b[0;32m~/git/ml-political-ads/helper.ipynb:63\u001b[0m, in \u001b[0;36mfind_words\u001b[0;34m(x, lexicon)\u001b[0m\n\u001b[1;32m      1\u001b[0m {\n\u001b[1;32m      2\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcells\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m      3\u001b[0m   {\n\u001b[1;32m      4\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      5\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m4\u001b[39m,\n\u001b[1;32m      6\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m88acc7d5\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m      7\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m      8\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m      9\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom bs4 import BeautifulSoup\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport pandas as pd\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     12\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport requests\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     13\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport re\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     14\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport json\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport spacy\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom spacy.matcher import PhraseMatcher\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     17\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimport numpy as np\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     18\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom youtube_transcript_api import YouTubeTranscriptApi\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     19\u001b[0m    ]\n\u001b[1;32m     20\u001b[0m   },\n\u001b[1;32m     21\u001b[0m   {\n\u001b[1;32m     22\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     23\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m5\u001b[39m,\n\u001b[1;32m     24\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m4c72fcd3\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     25\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     26\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     27\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     28\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_text_from_csv(fn): # get ad\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms text from csv file, return a dataframe with text, ad_id,ad_url, and report_url\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     29\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     30\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mText\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].reset_index(drop = True) # only get text ads\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_url = df[[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) # get the urls\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    urls = df_url[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].to_list() # create a list so we could get the report_urls\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     33\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    report_urls = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     34\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for url in urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        entity_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     36\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        creative_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     37\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://transparencyreport.google.com/transparencyreport/api/v3/politicalads/creatives/details?entity_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&creative_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&hl=en\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.format(entity_id,creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_urls.append(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     39\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    ad_text = [] # get texts from reports\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     40\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for report_url in report_urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     41\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        response = requests.get(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        text = response.text.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[0].split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        ad_text.append(text)  \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     44\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text = pd.DataFrame(ad_text).rename(columns = \u001b[39m\u001b[38;5;124m{\u001b[39m\u001b[38;5;124m0:\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m})\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_new = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mText\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m][[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     46\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.concat([df_text,df_new],axis=1)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplatform\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgoogle\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     48\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df  \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     49\u001b[0m    ]\n\u001b[1;32m     50\u001b[0m   },\n\u001b[1;32m     51\u001b[0m   {\n\u001b[1;32m     52\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     53\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m6\u001b[39m,\n\u001b[1;32m     54\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m5b928802\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     55\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     56\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     57\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     58\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# check the dataset \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     59\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef check_null(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].isnull().sum():\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        print (df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].isnull().sum())\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     62\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        df_text = df_text[df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].notnull()]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m---> 63\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        df_text.reset_index(drop=True,inplace=True)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     64\u001b[0m    ]\n\u001b[1;32m     65\u001b[0m   },\n\u001b[1;32m     66\u001b[0m   {\n\u001b[1;32m     67\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     68\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m7\u001b[39m,\n\u001b[1;32m     69\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0daaeae0\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     70\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     71\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     72\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     73\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# function that filter the urls and symbols in the text \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     74\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef clean_text(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     75\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    def filter_text(x):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     76\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp[s]?://\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mS+\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     77\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(url,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     78\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m[^\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mw\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124ms]\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x) # filter symbols\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     79\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        x = re.sub(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124ms+\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     80\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        ls=[w.lower() for w in x.split()] \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     81\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(ls)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     82\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].astype(str).apply(lambda x: filter_text(x))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     83\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].replace(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/u0026\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text.drop_duplicates(subset = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,keep = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     85\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text.reset_index(drop = True,inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_text\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     87\u001b[0m    ]\n\u001b[1;32m     88\u001b[0m   },\n\u001b[1;32m     89\u001b[0m   {\n\u001b[1;32m     90\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     91\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m8\u001b[39m,\n\u001b[1;32m     92\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma3c8ef8c\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     93\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m     94\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m     95\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m     96\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# the function that find the lexicon words in the text\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef find_words(x,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     98\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    topics= lexicon.keys()  \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     99\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    nlp = spacy.load(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m/usr/local/Cellar/jupyterlab/3.2.9/libexec/lib/python3.9/site-packages/en_core_web_lg/en_core_web_lg-3.2.0/\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    100\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    doc = nlp(x) # nlp() is spaCy 2.2 English language model \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    101\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    words= []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    102\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for t in topics:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    103\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matcher = PhraseMatcher(nlp.vocab)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        terms= lexicon[t]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    105\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        patterns = [nlp.make_doc(text) for text in terms]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    106\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matcher.add(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124mTerminologyList\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m, patterns) # spaCy2.2 phrase matcher\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    107\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        matches = matcher(doc)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    108\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        for match_id, start,end in matches:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    109\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            span = doc[start:end]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    110\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            words.append(span.text)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    111\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if words:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    112\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        words = list(set(words))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    113\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(words)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    else:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    115\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    116\u001b[0m    ]\n\u001b[1;32m    117\u001b[0m   },\n\u001b[1;32m    118\u001b[0m   {\n\u001b[1;32m    119\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    120\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m9\u001b[39m,\n\u001b[1;32m    121\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mff10e181\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    122\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    123\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    124\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    125\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# tagging the topic in each message\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    126\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef find_topic(x,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    127\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    topics= lexicon.keys()    \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    128\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if x==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    129\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m    \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    130\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if x != \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno words\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    131\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        words = x.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    132\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        labels = []        \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    133\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        for t in topics:            \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    134\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            terms = lexicon[t]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    135\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            if set(words)&set(terms):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    136\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                labels.append(t)                \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    137\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                #l = sorted(labels)        \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    138\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        return  \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(sorted(labels))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    139\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    140\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        #return \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.join(labels)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    141\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    142\u001b[0m    ]\n\u001b[1;32m    143\u001b[0m   },\n\u001b[1;32m    144\u001b[0m   {\n\u001b[1;32m    145\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    146\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m10\u001b[39m,\n\u001b[1;32m    147\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdc891547\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    148\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    149\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    150\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    151\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_word_lable(df_text,lexicon):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    152\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    with open (lexicon) as f:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    153\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        dic = json.load(f)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].astype(str).apply(lambda x: find_words(x,dic))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    155\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].apply(lambda x: find_topic(x,dic))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    156\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].apply(lambda x: \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mno topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m if x==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m else x)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    157\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_text\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    158\u001b[0m    ]\n\u001b[1;32m    159\u001b[0m   },\n\u001b[1;32m    160\u001b[0m   {\n\u001b[1;32m    161\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    162\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m11\u001b[39m,\n\u001b[1;32m    163\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m7f5e156b\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    164\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    165\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    166\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    167\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef count_topic(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    168\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mm_label\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].value_counts().rename_axis(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtopics\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(name=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    169\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_tag.assign(single_topic=df_tag[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtopics\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)).explode(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(drop = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    170\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag = df_tag.groupby(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_topic\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).sum().reset_index().sort_values(by = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, ascending = False)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    171\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_tag.reset_index(drop = True, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    172\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_tag\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    173\u001b[0m    ]\n\u001b[1;32m    174\u001b[0m   },\n\u001b[1;32m    175\u001b[0m   {\n\u001b[1;32m    176\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    177\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m12\u001b[39m,\n\u001b[1;32m    178\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m9be8a97d\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    179\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    180\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    181\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    182\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef count_word(df_text):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    183\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_text[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].value_counts().rename_axis(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(name=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    184\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_words.assign(single_word=df_words[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwords\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)).explode(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_word\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).reset_index(drop = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    185\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words = df_words.groupby(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msingle_word\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m).sum().reset_index().sort_values(by = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcounts\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, ascending = False)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    186\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_words.reset_index(drop = True, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    187\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_words\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    188\u001b[0m    ]\n\u001b[1;32m    189\u001b[0m   },\n\u001b[1;32m    190\u001b[0m   {\n\u001b[1;32m    191\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    192\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m13\u001b[39m,\n\u001b[1;32m    193\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbe9a9f65\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    194\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    195\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    196\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    197\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef read_fb(fn):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    198\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    199\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df.drop(columns = [\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mUnnamed: 0\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m],inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    200\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df.rename(columns = \u001b[39m\u001b[38;5;124m{\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbyline\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m},inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    201\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    pd.set_option(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdisplay.max_colwidth\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, None)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    202\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    facebook_df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mplatform\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfacebook\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    203\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return facebook_df\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    204\u001b[0m    ]\n\u001b[1;32m    205\u001b[0m   },\n\u001b[1;32m    206\u001b[0m   {\n\u001b[1;32m    207\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    208\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m14\u001b[39m,\n\u001b[1;32m    209\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m04528f75\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    210\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    211\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    212\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    213\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# go through the csv file, get youtube video\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ms ids from all video ads. Add new column \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    214\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# before running this function, make sure the csv file includes ad_url\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    215\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_youtube_id(fn): \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    216\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = pd.read_csv(fn)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    217\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df = df[df[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_type\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] == \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mVideo\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].reset_index(drop = True) # only get video ads\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    218\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_video = df[[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madvertiser_name\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mimpressions\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mspend_usd\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]].reset_index(drop = True) \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    219\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    urls = df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad_url\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].to_list() # \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    220\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    youtube_ids = [] # get youtube id\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    221\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    for url in urls:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    222\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        entity_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    223\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        creative_id = url.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[-1]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    224\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        report_url = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://transparencyreport.google.com/transparencyreport/api/v3/politicalads/creatives/details?entity_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&creative_id=\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m&hl=en\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.format(entity_id,creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    225\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        response = requests.get(report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    226\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        try:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    227\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            youtube_id = response.text.split(\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m)[3]\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    228\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            if len(list(youtube_id)) > 11: # mark ads violating google polices\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m                youtube_id = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id not available: this ad violated google ad policies.\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    230\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            #print(youtube_id + \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m + creative_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    231\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        except IndexError: # mark ads cannot be loaded\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    232\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            youtube_id = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id not available: cannot load the video with this ad_url.\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    233\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            #print(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124mcan\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt load this video, \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m + report_url)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    234\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m            pass\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        youtube_ids.append(youtube_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    236\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = youtube_ids\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    237\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    238\u001b[0m    ]\n\u001b[1;32m    239\u001b[0m   },\n\u001b[1;32m    240\u001b[0m   {\n\u001b[1;32m    241\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    242\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m15\u001b[39m,\n\u001b[1;32m    243\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3ac7ad49\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    244\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    245\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    246\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    247\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# check all videos, see which are available and drop duplicates. Add a new column video_available\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef check_video(df_video): \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    249\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  for id in df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    250\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    if len(list(id)) == 11:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    251\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video.drop_duplicates(subset = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,keep = \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfirst\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, inplace = True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    252\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video.reset_index(drop = True, inplace=True)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      yes_video = df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m].str.len() == 11\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    254\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mvideo_available\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = yes_video\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    255\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    256\u001b[0m    ]\n\u001b[1;32m    257\u001b[0m   },\n\u001b[1;32m    258\u001b[0m   {\n\u001b[1;32m    259\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcell_type\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    260\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexecution_count\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m16\u001b[39m,\n\u001b[1;32m    261\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbd8f2a85\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    262\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m    263\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutputs\u001b[39m\u001b[38;5;124m\"\u001b[39m: [],\n\u001b[1;32m    264\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[1;32m    265\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m# use youtube_id to get captions. Add a new column youtube_captions\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    266\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdef get_captions(df_video):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    267\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  youtube_captions = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    268\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  for youtube_id in df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124myoutube_id\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m]:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    try:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    270\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      subs = YouTubeTranscriptApi.get_transcript(youtube_id)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    271\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      alist = []\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    272\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      for sub in subs:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    273\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        alist.append(\u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;124m + sub[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m])\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    274\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      captions = \u001b[39m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\\"\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    275\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      for item in alist:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    276\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m        captions += item\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    277\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    except Exception as e:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    278\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m      captions = e   \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    279\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m    youtube_captions.append(captions)\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    280\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  #print(len(youtube_captions))\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    281\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  df_video[\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m] = youtube_captions\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m  return df_video\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    283\u001b[0m    ]\n\u001b[1;32m    284\u001b[0m   }\n\u001b[1;32m    285\u001b[0m  ],\n\u001b[1;32m    286\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    287\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minterpreter\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    288\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhash\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb2373ebc037b6b887ce6ae759e146cfea08225ef87b9ff98319d1ded9ec6eadc\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    289\u001b[0m   },\n\u001b[1;32m    290\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkernelspec\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    291\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdisplay_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPython 3 (ipykernel)\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    292\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlanguage\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    293\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython3\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    294\u001b[0m   },\n\u001b[1;32m    295\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlanguage_info\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    296\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcodemirror_mode\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m    297\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mipython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    298\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m3\u001b[39m\n\u001b[1;32m    299\u001b[0m    },\n\u001b[1;32m    300\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_extension\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.py\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    301\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmimetype\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext/x-python\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    302\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    303\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbconvert_exporter\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    304\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpygments_lexer\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mipython3\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    305\u001b[0m    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m3.9.12\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    306\u001b[0m   }\n\u001b[1;32m    307\u001b[0m  },\n\u001b[1;32m    308\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbformat\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m4\u001b[39m,\n\u001b[1;32m    309\u001b[0m  \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbformat_minor\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;241m5\u001b[39m\n\u001b[1;32m    310\u001b[0m }\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/ml-political-ads-gxQjH-2P/lib/python3.10/site-packages/spacy/__init__.py:51\u001b[0m, in \u001b[0;36mload\u001b[0;34m(name, vocab, disable, exclude, config)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload\u001b[39m(\n\u001b[1;32m     31\u001b[0m     name: Union[\u001b[38;5;28mstr\u001b[39m, Path],\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;241m*\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     36\u001b[0m     config: Union[Dict[\u001b[38;5;28mstr\u001b[39m, Any], Config] \u001b[38;5;241m=\u001b[39m util\u001b[38;5;241m.\u001b[39mSimpleFrozenDict(),\n\u001b[1;32m     37\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Language:\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;124;03m\"\"\"Load a spaCy model from an installed package or a local path.\u001b[39;00m\n\u001b[1;32m     39\u001b[0m \n\u001b[1;32m     40\u001b[0m \u001b[38;5;124;03m    name (str): Package name or model path.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[38;5;124;03m    RETURNS (Language): The loaded nlp object.\u001b[39;00m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 51\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mutil\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     52\u001b[0m \u001b[43m        \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvocab\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvocab\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdisable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdisable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexclude\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\n\u001b[1;32m     53\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/share/virtualenvs/ml-political-ads-gxQjH-2P/lib/python3.10/site-packages/spacy/util.py:427\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(name, vocab, disable, exclude, config)\u001b[0m\n\u001b[1;32m    425\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m OLD_MODEL_SHORTCUTS:\n\u001b[1;32m    426\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(Errors\u001b[38;5;241m.\u001b[39mE941\u001b[38;5;241m.\u001b[39mformat(name\u001b[38;5;241m=\u001b[39mname, full\u001b[38;5;241m=\u001b[39mOLD_MODEL_SHORTCUTS[name]))  \u001b[38;5;66;03m# type: ignore[index]\u001b[39;00m\n\u001b[0;32m--> 427\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(Errors\u001b[38;5;241m.\u001b[39mE050\u001b[38;5;241m.\u001b[39mformat(name\u001b[38;5;241m=\u001b[39mname))\n",
      "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model '/usr/local/Cellar/jupyterlab/3.2.9/libexec/lib/python3.9/site-packages/en_core_web_lg/en_core_web_lg-3.2.0/'. It doesn't seem to be a Python package or a valid path to a data directory."
     ]
    }
   ],
   "source": [
    "df_text = get_word_lable(df_text,'lexicon.json')\n",
    "df_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "53137728",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_text.to_csv('results/ad_ct.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "49112a37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_topic</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no topic</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>economic</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>education</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>governance</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>health</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>social programs</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>social and cultural</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>environment</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>safety</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>covid</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>foreign policy</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>immigration</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>donation</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           single_topic  counts\n",
       "0              no topic     102\n",
       "1              economic      41\n",
       "2             education      30\n",
       "3            governance      28\n",
       "4                health      25\n",
       "5       social programs      18\n",
       "6   social and cultural      14\n",
       "7           environment      12\n",
       "8                safety      11\n",
       "9                 covid       8\n",
       "10       foreign policy       8\n",
       "11          immigration       6\n",
       "12             donation       3"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_topic = count_topic(df_text)\n",
    "text_topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "55499592",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "      <th>words</th>\n",
       "      <th>m_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our democrat opponent outraised us last quarter so we need your help immediately the georgia polls are a dead heat so please help us close the fundraising gap today</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>our democrat opponent outraised us last quarter so we need your help immediately georgia will decide the senate majority in 2022 so help us close the fundraising gap</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>georgia will decide the senate majority in 2022 so help us close the fundraising gap our democrat opponent outraised us last quarter so we need your help immediately</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>tell gov lamont to stop putting innovation and access at risk send a letter here protect live saving research from gov lamont s government price setting proposal</td>\n",
       "      <td>PhRMA</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>there s no greater priority than defending democracy you can help</td>\n",
       "      <td>Democracy Initiative</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>ned lamont radical dems are letting criminals loose act to keep dangerous criminals off the streets</td>\n",
       "      <td>Connecticut Republican Party</td>\n",
       "      <td>facebook</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>richard blumenthal is failing the people of connecticut sign today to fire blumenthal in 2022</td>\n",
       "      <td>Connecticut Republican Party</td>\n",
       "      <td>facebook</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>sign the petition today if you want to replace richard blumenthal in 2022 act now</td>\n",
       "      <td>Connecticut Republican Party</td>\n",
       "      <td>facebook</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>last tuesday i had the privilege of appearing on fox 94 9 lee elci the voice of freedom here s a video in case you missed it</td>\n",
       "      <td>Robert Boris for State Representative</td>\n",
       "      <td>facebook</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>this is it the biggest test of my career in public service tonight we wrap up our fundraising period democrats across the state will use this information when deciding who to support at the critical state convention will our fundraising finally convince the establishment that this campaign is for real you can help answer that question by contributing now hilda</td>\n",
       "      <td>Santiago for Connecticut</td>\n",
       "      <td>facebook</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                           text  \\\n",
       "0                                                                                                                                                                                                          our democrat opponent outraised us last quarter so we need your help immediately the georgia polls are a dead heat so please help us close the fundraising gap today   \n",
       "1                                                                                                                                                                                                         our democrat opponent outraised us last quarter so we need your help immediately georgia will decide the senate majority in 2022 so help us close the fundraising gap   \n",
       "4                                                                                                                                                                                                         georgia will decide the senate majority in 2022 so help us close the fundraising gap our democrat opponent outraised us last quarter so we need your help immediately   \n",
       "5                                                                                                                                                                                                             tell gov lamont to stop putting innovation and access at risk send a letter here protect live saving research from gov lamont s government price setting proposal   \n",
       "7                                                                                                                                                                                                                                                                                                             there s no greater priority than defending democracy you can help   \n",
       "..                                                                                                                                                                                                                                                                                                                                                                          ...   \n",
       "225                                                                                                                                                                                                                                                                         ned lamont radical dems are letting criminals loose act to keep dangerous criminals off the streets   \n",
       "226                                                                                                                                                                                                                                                                               richard blumenthal is failing the people of connecticut sign today to fire blumenthal in 2022   \n",
       "227                                                                                                                                                                                                                                                                                           sign the petition today if you want to replace richard blumenthal in 2022 act now   \n",
       "229                                                                                                                                                                                                                                                last tuesday i had the privilege of appearing on fox 94 9 lee elci the voice of freedom here s a video in case you missed it   \n",
       "230  this is it the biggest test of my career in public service tonight we wrap up our fundraising period democrats across the state will use this information when deciding who to support at the critical state convention will our fundraising finally convince the establishment that this campaign is for real you can help answer that question by contributing now hilda   \n",
       "\n",
       "                           advertiser_name  platform     words   m_label  \n",
       "0                      Team Herschel, Inc.    google  no words  no topic  \n",
       "1                      Team Herschel, Inc.    google  no words  no topic  \n",
       "4                      Team Herschel, Inc.    google  no words  no topic  \n",
       "5                                    PhRMA    google  no words  no topic  \n",
       "7                     Democracy Initiative    google  no words  no topic  \n",
       "..                                     ...       ...       ...       ...  \n",
       "225           Connecticut Republican Party  facebook  no words  no topic  \n",
       "226           Connecticut Republican Party  facebook  no words  no topic  \n",
       "227           Connecticut Republican Party  facebook  no words  no topic  \n",
       "229  Robert Boris for State Representative  facebook  no words  no topic  \n",
       "230               Santiago for Connecticut  facebook  no words  no topic  \n",
       "\n",
       "[102 rows x 5 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_text[df_text['m_label']=='no topic']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f957f052",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_word</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no words</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>education</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>health</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tax</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>public schools</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>natural gas</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>deductibles</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>deficit</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>loan assistance</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>job growth</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>130 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         single_word  counts\n",
       "0           no words     102\n",
       "1          education      17\n",
       "2             health      15\n",
       "3                tax      15\n",
       "4     public schools      13\n",
       "..               ...     ...\n",
       "125      natural gas       1\n",
       "126      deductibles       1\n",
       "127          deficit       1\n",
       "128  loan assistance       1\n",
       "129       job growth       1\n",
       "\n",
       "[130 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_word = count_word(df_text)\n",
    "text_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "68c771bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "School and State Finance Project                                            25\n",
       "NED FOR CT                                                                  18\n",
       "Connecticut Republican Party                                                11\n",
       "50CAN, INC.                                                                 10\n",
       "Bob for Governor - David Becker, Treasurer - Approved by Bob Stefanowski     9\n",
       "                                                                            ..\n",
       "Andrew Adam Theriac                                                          1\n",
       "Financial Study Network                                                      1\n",
       "CT State Employee Bargaining Agent Coalition                                 1\n",
       "Jacobs & Jacobs                                                              1\n",
       "Lumaj for Senate                                                             1\n",
       "Name: advertiser_name, Length: 68, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_text['advertiser_name'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31790b74",
   "metadata": {},
   "source": [
    "# GOOGLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d93f1094",
   "metadata": {},
   "outputs": [],
   "source": [
    "if google_df['text'].isnull().sum():\n",
    "    print (google_df['text'].isnull().sum())\n",
    "    google_df = google_df[google_df['text'].notnull()]\n",
    "    google_df.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "63cc09f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 28 entries, 0 to 27\n",
      "Data columns (total 3 columns):\n",
      " #   Column           Non-Null Count  Dtype \n",
      "---  ------           --------------  ----- \n",
      " 0   text             28 non-null     object\n",
      " 1   advertiser_name  28 non-null     object\n",
      " 2   platform         28 non-null     object\n",
      "dtypes: object(3)\n",
      "memory usage: 800.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "google_df = clean_text(google_df)\n",
    "google_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "730f868c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "      <th>words</th>\n",
       "      <th>m_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>our democrat opponent outraised us last quarter so we need your help immediately the georgia polls are a dead heat so please help us close the fundraising gap today</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>our democrat opponent outraised us last quarter so we need your help immediately georgia will decide the senate majority in 2022 so help us close the fundraising gap</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>it s time to focus on the hard working families in connecticut and end the d c power grab george logan is the republican leadership washington and connecticut need donate today</td>\n",
       "      <td>GEORGE LOGAN FOR CONGRESS</td>\n",
       "      <td>google</td>\n",
       "      <td>donate</td>\n",
       "      <td>donation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>help herschel walker reach his critical end of quarter fundraising goal donate to the official website of herschel walker for united states senate today</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>donate</td>\n",
       "      <td>donation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>georgia will decide the senate majority in 2022 so help us close the fundraising gap our democrat opponent outraised us last quarter so we need your help immediately</td>\n",
       "      <td>Team Herschel, Inc.</td>\n",
       "      <td>google</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                               text  \\\n",
       "0              our democrat opponent outraised us last quarter so we need your help immediately the georgia polls are a dead heat so please help us close the fundraising gap today   \n",
       "1             our democrat opponent outraised us last quarter so we need your help immediately georgia will decide the senate majority in 2022 so help us close the fundraising gap   \n",
       "2  it s time to focus on the hard working families in connecticut and end the d c power grab george logan is the republican leadership washington and connecticut need donate today   \n",
       "3                          help herschel walker reach his critical end of quarter fundraising goal donate to the official website of herschel walker for united states senate today   \n",
       "4             georgia will decide the senate majority in 2022 so help us close the fundraising gap our democrat opponent outraised us last quarter so we need your help immediately   \n",
       "\n",
       "             advertiser_name platform     words   m_label  \n",
       "0        Team Herschel, Inc.   google  no words  no topic  \n",
       "1        Team Herschel, Inc.   google  no words  no topic  \n",
       "2  GEORGE LOGAN FOR CONGRESS   google    donate  donation  \n",
       "3        Team Herschel, Inc.   google    donate  donation  \n",
       "4        Team Herschel, Inc.   google  no words  no topic  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "google_df = get_word_lable(google_df,'lexicon.json')\n",
    "google_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "390668c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_topic</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no topic</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>governance</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>economic</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>donation</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>social programs</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>covid</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>immigration</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      single_topic  counts\n",
       "0         no topic      15\n",
       "1       governance       6\n",
       "2         economic       3\n",
       "3         donation       2\n",
       "4  social programs       2\n",
       "5            covid       1\n",
       "6      immigration       1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "google_topic = count_topic(google_df)\n",
    "google_topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d513975f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_word</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no words</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>filibuster</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>donate</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>laws</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>border</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>covid</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>jobs</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>out of pocket</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>out of pocket costs</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>savings</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>tax</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>taxes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>vaccines</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            single_word  counts\n",
       "0              no words      15\n",
       "1            filibuster       4\n",
       "2                donate       2\n",
       "3                  laws       2\n",
       "4                border       1\n",
       "5                 covid       1\n",
       "6                  jobs       1\n",
       "7         out of pocket       1\n",
       "8   out of pocket costs       1\n",
       "9               savings       1\n",
       "10                  tax       1\n",
       "11                taxes       1\n",
       "12             vaccines       1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "google_word = count_word(google_df)\n",
    "google_word"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12971218",
   "metadata": {},
   "source": [
    "# FACEBOOK "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ec652bbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    }
   ],
   "source": [
    "if facebook_df['text'].isnull().sum():\n",
    "    print (facebook_df['text'].isnull().sum())\n",
    "    facebook_df = facebook_df[facebook_df['text'].notnull()]\n",
    "    facebook_df.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "949bad7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yzhao/git/ml-political-ads/helper.ipynb:52: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"id\": \"5b928802\",\n",
      "/Users/yzhao/git/ml-political-ads/helper.ipynb:53: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"metadata\": {},\n",
      "/Users/yzhao/git/ml-political-ads/helper.ipynb:54: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"outputs\": [],\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method DataFrame.info of                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 text  \\\n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                          please join cea s red for ed day of action this wednesday come to hartford to meet with your legislators or if you can t make it take just a few minutes to call or email   \n",
       "1                                                                                                                                                                                                                                                  thinking about attending a connecticut state community college get all the information and support you need during our virtual showcases april 28 30th hear about community college and program options throughout the state transfer opportunities financial aid careers and more all from your couch live chat will be available register today by visiting the link in our bio   \n",
       "2                                                                                                                                                                                                                                                                                                                                                                     governor lamont is protecting abortion rights in connecticut the governor is set to sign a major piece of legislation protecting abortion ahead of an upcoming supreme court ruling sign up now to learn more about what the governor is doing for connecticut   \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                             hey connecticut have you heard governor lamont has signed a bill suspending the gas tax this means real relief for working families across the state add your name now to get more updates on what gov lamont is doing for connecticut   \n",
       "4                                                                                                                                                                                                                                                                                                                                      cost increases are hitting families in connecticut that s why governor lamont and the legislature have suspended the gas tax it ll cost you less to fill up at the pump that s real relief for connecticut families add your name and find out more about how the governor is working for you   \n",
       "..                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               ...   \n",
       "199                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      drugs flood through the border killing our families and neighbors sign to save our families   \n",
       "200                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     last tuesday i had the privilege of appearing on fox 94 9 lee elci the voice of freedom here s a video in case you missed it   \n",
       "201                                                                                                                                                                                                                                                       this is it the biggest test of my career in public service tonight we wrap up our fundraising period democrats across the state will use this information when deciding who to support at the critical state convention will our fundraising finally convince the establishment that this campaign is for real you can help answer that question by contributing now hilda   \n",
       "202                                                                                                                                                                                                            ctteacher laura baker coronis got an unexpected surprise today when milken educator awards senior vice president jane foley called her name as this season s ct milken educator award recipient the ansonia high school community went wild to celebrate the math teacher ansonia public schools connecticut state department of education office of lt governor susan bysiewicz educatect ansonia ansoniact goodnews   \n",
       "203  ansonia high school math teacher laura baker coronis is the sole connecticut milken educator award recipient this year thank you to connecticut commissioner of education charlene russell tucker and office of lt governor susan bysiewicz for joining us to surprise laura with the award in front of appreciative students and staff this morning follow along to see some of the photos from her big day learn more about laura baker coronis laura coronis ansonia high school ansonia public schools connecticut state department of education educatect ctteachers southernctalumni ansonia ansoniact teachers education   \n",
       "\n",
       "                           advertiser_name  platform  \n",
       "0        CONNECTICUT EDUCATION ASSOCIATION  facebook  \n",
       "1                         odonnell company  facebook  \n",
       "2                               NED FOR CT  facebook  \n",
       "3                               NED FOR CT  facebook  \n",
       "4                               NED FOR CT  facebook  \n",
       "..                                     ...       ...  \n",
       "199           Connecticut Republican Party  facebook  \n",
       "200  Robert Boris for State Representative  facebook  \n",
       "201               Santiago for Connecticut  facebook  \n",
       "202               MILKEN FAMILY FOUNDATION  facebook  \n",
       "203               MILKEN FAMILY FOUNDATION  facebook  \n",
       "\n",
       "[204 rows x 3 columns]>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebook_df = clean_text(facebook_df)\n",
    "facebook_df.info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0f391f4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yzhao/git/ml-political-ads/helper.ipynb:100: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"    for t in topics:\\n\",\n",
      "/Users/yzhao/git/ml-political-ads/helper.ipynb:101: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"        matcher = PhraseMatcher(nlp.vocab)\\n\",\n",
      "/Users/yzhao/git/ml-political-ads/helper.ipynb:102: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"        terms= lexicon[t]\\n\",\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>platform</th>\n",
       "      <th>words</th>\n",
       "      <th>m_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>please join cea s red for ed day of action this wednesday come to hartford to meet with your legislators or if you can t make it take just a few minutes to call or email</td>\n",
       "      <td>CONNECTICUT EDUCATION ASSOCIATION</td>\n",
       "      <td>facebook</td>\n",
       "      <td>no words</td>\n",
       "      <td>no topic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>thinking about attending a connecticut state community college get all the information and support you need during our virtual showcases april 28 30th hear about community college and program options throughout the state transfer opportunities financial aid careers and more all from your couch live chat will be available register today by visiting the link in our bio</td>\n",
       "      <td>odonnell company</td>\n",
       "      <td>facebook</td>\n",
       "      <td>college,community college</td>\n",
       "      <td>social programs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>governor lamont is protecting abortion rights in connecticut the governor is set to sign a major piece of legislation protecting abortion ahead of an upcoming supreme court ruling sign up now to learn more about what the governor is doing for connecticut</td>\n",
       "      <td>NED FOR CT</td>\n",
       "      <td>facebook</td>\n",
       "      <td>supreme court,abortion</td>\n",
       "      <td>governance,health,social and cultural</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hey connecticut have you heard governor lamont has signed a bill suspending the gas tax this means real relief for working families across the state add your name now to get more updates on what gov lamont is doing for connecticut</td>\n",
       "      <td>NED FOR CT</td>\n",
       "      <td>facebook</td>\n",
       "      <td>tax</td>\n",
       "      <td>economic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cost increases are hitting families in connecticut that s why governor lamont and the legislature have suspended the gas tax it ll cost you less to fill up at the pump that s real relief for connecticut families add your name and find out more about how the governor is working for you</td>\n",
       "      <td>NED FOR CT</td>\n",
       "      <td>facebook</td>\n",
       "      <td>tax</td>\n",
       "      <td>economic</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                text  \\\n",
       "0                                                                                                                                                                                                          please join cea s red for ed day of action this wednesday come to hartford to meet with your legislators or if you can t make it take just a few minutes to call or email   \n",
       "1  thinking about attending a connecticut state community college get all the information and support you need during our virtual showcases april 28 30th hear about community college and program options throughout the state transfer opportunities financial aid careers and more all from your couch live chat will be available register today by visiting the link in our bio   \n",
       "2                                                                                                                     governor lamont is protecting abortion rights in connecticut the governor is set to sign a major piece of legislation protecting abortion ahead of an upcoming supreme court ruling sign up now to learn more about what the governor is doing for connecticut   \n",
       "3                                                                                                                                             hey connecticut have you heard governor lamont has signed a bill suspending the gas tax this means real relief for working families across the state add your name now to get more updates on what gov lamont is doing for connecticut   \n",
       "4                                                                                      cost increases are hitting families in connecticut that s why governor lamont and the legislature have suspended the gas tax it ll cost you less to fill up at the pump that s real relief for connecticut families add your name and find out more about how the governor is working for you   \n",
       "\n",
       "                     advertiser_name  platform                      words  \\\n",
       "0  CONNECTICUT EDUCATION ASSOCIATION  facebook                   no words   \n",
       "1                   odonnell company  facebook  college,community college   \n",
       "2                         NED FOR CT  facebook     supreme court,abortion   \n",
       "3                         NED FOR CT  facebook                        tax   \n",
       "4                         NED FOR CT  facebook                        tax   \n",
       "\n",
       "                                 m_label  \n",
       "0                               no topic  \n",
       "1                        social programs  \n",
       "2  governance,health,social and cultural  \n",
       "3                               economic  \n",
       "4                               economic  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebook_df = get_word_lable(facebook_df,'lexicon.json')\n",
    "facebook_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d24d3f3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_topic</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no topic</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>economic</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>education</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>health</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>governance</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>social programs</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>social and cultural</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>environment</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>safety</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>foreign policy</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>covid</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>immigration</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>donation</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           single_topic  counts\n",
       "0              no topic      86\n",
       "1              economic      38\n",
       "2             education      30\n",
       "3                health      25\n",
       "4            governance      22\n",
       "5       social programs      16\n",
       "6   social and cultural      14\n",
       "7           environment      12\n",
       "8                safety      11\n",
       "9        foreign policy       8\n",
       "10                covid       7\n",
       "11          immigration       5\n",
       "12             donation       1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebook_topic = count_topic(facebook_df)\n",
    "facebook_topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "88bda7d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>single_word</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>no words</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>education</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>health</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tax</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>public schools</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>loan assistance</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>living wage</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>covid 19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>deductibles</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>job growth</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>126 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         single_word  counts\n",
       "0           no words      86\n",
       "1          education      17\n",
       "2             health      15\n",
       "3                tax      14\n",
       "4     public schools      13\n",
       "..               ...     ...\n",
       "121  loan assistance       1\n",
       "122      living wage       1\n",
       "123         covid 19       1\n",
       "124      deductibles       1\n",
       "125       job growth       1\n",
       "\n",
       "[126 rows x 2 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "facebook_word = count_word(facebook_df)\n",
    "facebook_word"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b2373ebc037b6b887ce6ae759e146cfea08225ef87b9ff98319d1ded9ec6eadc"
  },
  "kernelspec": {
   "display_name": "Python 3.10.0 64-bit ('ml-political-ads-gxQjH-2P': pipenv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}