02.Data Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cc2967a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import required depdencies\n",
    "\n",
    "import pandas as pd\n",
    "pd.options.mode.chained_assignment=None  # default='warn'\n",
    "import re\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e5ee8a82",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "      <th>from</th>\n",
       "      <th>to</th>\n",
       "      <th>subject</th>\n",
       "      <th>x-from</th>\n",
       "      <th>x-to</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>tim.belden@enron.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>john.lavorato@enron.com</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>leah.arsdall@enron.com</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>randall.gay@enron.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>greg.piper@enron.com</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>2001-11-28 13:30:11</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>kori.loibl@enron.com</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Loibl, Kori &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/CN=K...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>2001-11-28 12:47:48</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>john.lavorato@enron.com</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Lavorato, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>2001-11-28 07:20:00</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>dawn.doucet@enron.com</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Doucet, Dawn &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/CN=...</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>2001-11-27 11:52:45</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>jeanie.slone@enron.com</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Slone, Jeanie &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/CN...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>2001-11-26 10:48:43</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>livia_zufferli@monitor.com</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>'Livia_Zufferli@Monitor.com@ENRON'</td>\n",
       "      <td>575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  content  \\\n",
       "0                                    Here is our forecast   \n",
       "1       Traveling to have a business meeting takes the...   \n",
       "2                          test successful.  way to go!!!   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...   \n",
       "4                       Let's shoot for Tuesday at 11:45.   \n",
       "...                                                   ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...   \n",
       "495543  Some of my position is with the Alberta Term b...   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...   \n",
       "495546  i think the YMCA has a class that is for peopl...   \n",
       "\n",
       "                       date                     from  \\\n",
       "0       2001-05-14 16:39:00  phillip.allen@enron.com   \n",
       "1       2001-05-04 13:51:00  phillip.allen@enron.com   \n",
       "2       2000-10-18 03:00:00  phillip.allen@enron.com   \n",
       "3       2000-10-23 06:13:00  phillip.allen@enron.com   \n",
       "4       2000-08-31 05:07:00  phillip.allen@enron.com   \n",
       "...                     ...                      ...   \n",
       "495542  2001-11-28 13:30:11  john.zufferli@enron.com   \n",
       "495543  2001-11-28 12:47:48  john.zufferli@enron.com   \n",
       "495544  2001-11-28 07:20:00  john.zufferli@enron.com   \n",
       "495545  2001-11-27 11:52:45  john.zufferli@enron.com   \n",
       "495546  2001-11-26 10:48:43  john.zufferli@enron.com   \n",
       "\n",
       "                                to                    subject  \\\n",
       "0             tim.belden@enron.com                        NaN   \n",
       "1          john.lavorato@enron.com                        Re:   \n",
       "2           leah.arsdall@enron.com                   Re: test   \n",
       "3            randall.gay@enron.com                        NaN   \n",
       "4             greg.piper@enron.com                  Re: Hello   \n",
       "...                            ...                        ...   \n",
       "495542        kori.loibl@enron.com   Trade with John Lavorato   \n",
       "495543     john.lavorato@enron.com                 Gas Hedges   \n",
       "495544       dawn.doucet@enron.com           RE: CONFIDENTIAL   \n",
       "495545      jeanie.slone@enron.com  Calgary Analyst/Associate   \n",
       "495546  livia_zufferli@monitor.com           RE: ali's essays   \n",
       "\n",
       "                                                   x-from  \\\n",
       "0                                         Phillip K Allen   \n",
       "1                                         Phillip K Allen   \n",
       "2                                         Phillip K Allen   \n",
       "3                                         Phillip K Allen   \n",
       "4                                         Phillip K Allen   \n",
       "...                                                   ...   \n",
       "495542  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495543  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495544  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495545  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495546  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "\n",
       "                                                     x-to  n_words  \n",
       "0                Tim Belden <Tim Belden/Enron@EnronXGate>        4  \n",
       "1       John J Lavorato <John J Lavorato/ENRON@enronXg...      139  \n",
       "2                                        Leah Van Arsdall        5  \n",
       "3                                           Randall L Gay       34  \n",
       "4                                              Greg Piper        6  \n",
       "...                                                   ...      ...  \n",
       "495542  Loibl, Kori </O=ENRON/OU=NA/CN=RECIPIENTS/CN=K...       48  \n",
       "495543  Lavorato, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...       23  \n",
       "495544  Doucet, Dawn </O=ENRON/OU=NA/CN=RECIPIENTS/CN=...       54  \n",
       "495545  Slone, Jeanie </O=ENRON/OU=NA/CN=RECIPIENTS/CN...       27  \n",
       "495546                 'Livia_Zufferli@Monitor.com@ENRON'      575  \n",
       "\n",
       "[495547 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/metadata.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce563844",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "      <th>from</th>\n",
       "      <th>to</th>\n",
       "      <th>subject</th>\n",
       "      <th>x-from</th>\n",
       "      <th>x-to</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>tim.belden@enron.com</td>\n",
       "      <td></td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>john.lavorato@enron.com</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>leah.arsdall@enron.com</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>randall.gay@enron.com</td>\n",
       "      <td></td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>phillip.allen@enron.com</td>\n",
       "      <td>greg.piper@enron.com</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>2001-11-28 13:30:11</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>kori.loibl@enron.com</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Loibl, Kori &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/CN=K...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>2001-11-28 12:47:48</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>john.lavorato@enron.com</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Lavorato, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>2001-11-28 07:20:00</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>dawn.doucet@enron.com</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Doucet, Dawn &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/CN=...</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>2001-11-27 11:52:45</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>jeanie.slone@enron.com</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>Slone, Jeanie &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/CN...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>2001-11-26 10:48:43</td>\n",
       "      <td>john.zufferli@enron.com</td>\n",
       "      <td>livia_zufferli@monitor.com</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>Zufferli, John &lt;/O=ENRON/OU=NA/CN=RECIPIENTS/C...</td>\n",
       "      <td>'Livia_Zufferli@Monitor.com@ENRON'</td>\n",
       "      <td>575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  content  \\\n",
       "0                                    Here is our forecast   \n",
       "1       Traveling to have a business meeting takes the...   \n",
       "2                          test successful.  way to go!!!   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...   \n",
       "4                       Let's shoot for Tuesday at 11:45.   \n",
       "...                                                   ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...   \n",
       "495543  Some of my position is with the Alberta Term b...   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...   \n",
       "495546  i think the YMCA has a class that is for peopl...   \n",
       "\n",
       "                       date                     from  \\\n",
       "0       2001-05-14 16:39:00  phillip.allen@enron.com   \n",
       "1       2001-05-04 13:51:00  phillip.allen@enron.com   \n",
       "2       2000-10-18 03:00:00  phillip.allen@enron.com   \n",
       "3       2000-10-23 06:13:00  phillip.allen@enron.com   \n",
       "4       2000-08-31 05:07:00  phillip.allen@enron.com   \n",
       "...                     ...                      ...   \n",
       "495542  2001-11-28 13:30:11  john.zufferli@enron.com   \n",
       "495543  2001-11-28 12:47:48  john.zufferli@enron.com   \n",
       "495544  2001-11-28 07:20:00  john.zufferli@enron.com   \n",
       "495545  2001-11-27 11:52:45  john.zufferli@enron.com   \n",
       "495546  2001-11-26 10:48:43  john.zufferli@enron.com   \n",
       "\n",
       "                                to                    subject  \\\n",
       "0             tim.belden@enron.com                              \n",
       "1          john.lavorato@enron.com                        Re:   \n",
       "2           leah.arsdall@enron.com                   Re: test   \n",
       "3            randall.gay@enron.com                              \n",
       "4             greg.piper@enron.com                  Re: Hello   \n",
       "...                            ...                        ...   \n",
       "495542        kori.loibl@enron.com   Trade with John Lavorato   \n",
       "495543     john.lavorato@enron.com                 Gas Hedges   \n",
       "495544       dawn.doucet@enron.com           RE: CONFIDENTIAL   \n",
       "495545      jeanie.slone@enron.com  Calgary Analyst/Associate   \n",
       "495546  livia_zufferli@monitor.com           RE: ali's essays   \n",
       "\n",
       "                                                   x-from  \\\n",
       "0                                         Phillip K Allen   \n",
       "1                                         Phillip K Allen   \n",
       "2                                         Phillip K Allen   \n",
       "3                                         Phillip K Allen   \n",
       "4                                         Phillip K Allen   \n",
       "...                                                   ...   \n",
       "495542  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495543  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495544  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495545  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "495546  Zufferli, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...   \n",
       "\n",
       "                                                     x-to  n_words  \n",
       "0                Tim Belden <Tim Belden/Enron@EnronXGate>        4  \n",
       "1       John J Lavorato <John J Lavorato/ENRON@enronXg...      139  \n",
       "2                                        Leah Van Arsdall        5  \n",
       "3                                           Randall L Gay       34  \n",
       "4                                              Greg Piper        6  \n",
       "...                                                   ...      ...  \n",
       "495542  Loibl, Kori </O=ENRON/OU=NA/CN=RECIPIENTS/CN=K...       48  \n",
       "495543  Lavorato, John </O=ENRON/OU=NA/CN=RECIPIENTS/C...       23  \n",
       "495544  Doucet, Dawn </O=ENRON/OU=NA/CN=RECIPIENTS/CN=...       54  \n",
       "495545  Slone, Jeanie </O=ENRON/OU=NA/CN=RECIPIENTS/CN...       27  \n",
       "495546                 'Livia_Zufferli@Monitor.com@ENRON'      575  \n",
       "\n",
       "[495547 rows x 9 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace NaN with empty string\n",
    "\n",
    "df = df.fillna(\"\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d3116747",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \n",
       "0                                    Here is our forecast        4  \n",
       "1       Traveling to have a business meeting takes the...      139  \n",
       "2                          test successful.  way to go!!!        5  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6  \n",
       "...                                                   ...      ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48  \n",
       "495543  Some of my position is with the Alberta Term b...       23  \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27  \n",
       "495546  i think the YMCA has a class that is for peopl...      575  \n",
       "\n",
       "[495547 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Keep only message_id, subject, content, n_words \n",
    "\n",
    "df = df[['message_id', \"subject\", 'content', 'n_words']]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2767d55",
   "metadata": {},
   "source": [
    "### Data Cleansing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "35951b3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                                         18822\n",
       "RE:                                                       6458\n",
       "Re:                                                       6283\n",
       "Demand Ken Lay Donate Proceeds from Enron Stock Sales     1124\n",
       "FW:                                                        933\n",
       "                                                         ...  \n",
       "working again                                                1\n",
       "FW: [JOKE FACTORY]  Serves you right...                      1\n",
       "Fwd: Good Advice                                             1\n",
       "invasion of privacy                                          1\n",
       "Calgary Analyst/Associate                                    1\n",
       "Name: subject, Length: 152895, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check email subject types\n",
    "\n",
    "df[\"subject\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c0b57bd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  \n",
       "1       Traveling to have a business meeting takes the...      139  \n",
       "2                          test successful.  way to go!!!        5  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9  \n",
       "7                        any morning between 10 and 11:30        6  \n",
       "...                                                   ...      ...  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75  \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137  \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413  \n",
       "\n",
       "[103469 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get replied emails\n",
    "\n",
    "re_emails = df[df[\"subject\"].str.contains(\"Re:\")]\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2bd9127a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    103469.000000\n",
       "mean        203.782795\n",
       "std         326.605542\n",
       "min           1.000000\n",
       "25%          45.000000\n",
       "50%         124.000000\n",
       "75%         262.000000\n",
       "max       27651.000000\n",
       "Name: n_words, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check statistics for replied emails\n",
    "\n",
    "re_emails['n_words'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c80f372e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>&lt;20364402.1075855690481.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>no</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>&lt;2068674.1075855691152.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: #30</td>\n",
       "      <td>2000-1969=31</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>571</th>\n",
       "      <td>&lt;23861783.1075855732328.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: You Game?</td>\n",
       "      <td>raincheck?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751</th>\n",
       "      <td>&lt;262584.1075855670688.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>no</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>784</th>\n",
       "      <td>&lt;30355239.1075855671370.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: #30</td>\n",
       "      <td>2000-1969=31</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478835</th>\n",
       "      <td>&lt;30544682.1075851696731.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Howdy</td>\n",
       "      <td>Saw</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480362</th>\n",
       "      <td>&lt;33444164.1075858067071.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: SFA update</td>\n",
       "      <td>yes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480743</th>\n",
       "      <td>&lt;21773667.1075858024147.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: SFA update</td>\n",
       "      <td>yes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483223</th>\n",
       "      <td>&lt;10455020.1075858063059.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: SFA update</td>\n",
       "      <td>yes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488582</th>\n",
       "      <td>&lt;6460284.1075839930003.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Model for 04-19</td>\n",
       "      <td>Thanks</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>677 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id              subject  \\\n",
       "160     <20364402.1075855690481.JavaMail.evans@thyme>                  Re:   \n",
       "194      <2068674.1075855691152.JavaMail.evans@thyme>              Re: #30   \n",
       "571     <23861783.1075855732328.JavaMail.evans@thyme>        Re: You Game?   \n",
       "751       <262584.1075855670688.JavaMail.evans@thyme>                  Re:   \n",
       "784     <30355239.1075855671370.JavaMail.evans@thyme>              Re: #30   \n",
       "...                                               ...                  ...   \n",
       "478835  <30544682.1075851696731.JavaMail.evans@thyme>            Re: Howdy   \n",
       "480362  <33444164.1075858067071.JavaMail.evans@thyme>       Re: SFA update   \n",
       "480743  <21773667.1075858024147.JavaMail.evans@thyme>       Re: SFA update   \n",
       "483223  <10455020.1075858063059.JavaMail.evans@thyme>       Re: SFA update   \n",
       "488582   <6460284.1075839930003.JavaMail.evans@thyme>  Re: Model for 04-19   \n",
       "\n",
       "             content  n_words  \n",
       "160               no        1  \n",
       "194     2000-1969=31        1  \n",
       "571       raincheck?        1  \n",
       "751               no        1  \n",
       "784     2000-1969=31        1  \n",
       "...              ...      ...  \n",
       "478835           Saw        1  \n",
       "480362           yes        1  \n",
       "480743           yes        1  \n",
       "483223           yes        1  \n",
       "488582        Thanks        1  \n",
       "\n",
       "[677 rows x 4 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check replied emails with 1 word\n",
    "\n",
    "re_emails[re_emails['n_words'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "84e4d316",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pretty printing has been turned OFF\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['no', '2000-1969=31', 'raincheck?', 'no', '2000-1969=31', 'raincheck?', 'no', '2000-1969=31', 'raincheck?', 'ok', '83', 'ok', '83', 'ok', '83', 'Thanks', 'Thanks', 'Thanks', 'Thanks', 'Thanks', 'bujy6666666jhhb', 'bujy6666666jhhb', 'bujy6666666jhhb', 'thanks', 'thanks', 'thanks', 'excellent!!!!!', 'fine', 'fine', 'excellent!!!!!', 'excellent!!!!!', 'fine', 'ok', 'NO', 'test', 'test', 'NO', 'ok', 'test', 'NO', 'ok', 'dirty!!!!!', 'yes', '713-853-6203', 'yes', '713-853-6203', 'dirty!!!!!', '713-853-6203', 'dirty!!!!!', 'yes', 'Why', 'Agreed.', 'Thanks', 'Amen.', 'Huh?', 'Fine.', 'o.k.', 'Thanks.', 'thanks.', 'SF', 'ok.', 'agreed.', 'thanks', 'Attachment?', 'Great.', '415.782.7854.', 'Alberta?', 'yes', 'nada.', 'Yes.', 'yep.', 'yes.', '503-464-8058', 'Roger.', 'yes', 'Fine.', 'o.k.', 'Thanks', '503-464-8058', 'agreed.', 'yes.', 'nope.', '415.782.7854.', 'Alberta?', 'agreed.', 'thanks', 'Attachment?', 'Great.', 'nada.', 'Thanks.', 'Yes.', 'thanks.', 'yep.', 'yes.', 'Roger.', 'Agreed.', 'Amen.', 'Huh?', 'SF', 'ok.', 'fine', 'Thanks', 'ok', 'yes', '??????', 'no', 'fine', 'Thanks', 'ok', 'yes', '??????', 'no', 'fine', 'Thanks', 'ok', 'yes', '??????', 'no', 'Thanks.', 'Yes.', 'Thanks', 'Thanks.', 'Yes.', 'Thanks', 'no', 'Thanx!', '403-343-0911?', 'Perhaps...', 'Thanx!', '403-343-0911?', 'no', 'Perhaps...', 'no', 'no', 'no', 'Thanks.', 'done', 'unsubscribe', 'done', '281.343.8233', 'unsubscribe', 'Thanks.', '281.343.8233', 'done', 'Thanks.', '281.343.8233', 'Thanks.', 'done', 'unsubscribe', 'ok', 'Thanks', 'OK', 'sure', 'sure', 'ok', 'Thanks', 'OK', 'sure', 'ok', 'Thanks', 'OK', 'approved', 'thanks', 'congratulations!!!!', 'enjoy!!!', '???????????????????????', 'none', 'yes', 'Done.', 'thanks', 'Kewel!!!', 'Thanks.', 'Nope.', 'Done.', 'thanks', 'Kewel!!!', 'Thanks.', 'Thanks...Billy', 'Nope.', 'Thanks...Billy', 'Thanks.', 'thanks', 'Kewel!!!', 'Thanks.', 'Nope.', 'Done.', 'OK', 'OK', 'OK', 'ty', 'ty', 'ty', '503-464-3740', '503-464-3740', 'Thanks.', 'OK', 'yes.', 'Fish.', 'Thanks!', 'large', 'non-smoking', 'yes', 'Thanks.', 'OK', 'yes.', 'Fish.', 'Thanks!', 'large', 'non-smoking', 'yes', 'SOLD', 'shendri', 'shendri', 'Thanks', 'OK..', 'Yes', 'Thanks', 'Yes', 'OK..', 'Yes', 'OK..', 'Thanks', 'Thanks!', 'ok', 'ok', 'thanks', 'Ughh...', 'Roger.', 'Nope.', 'Done.', 'Nope...', 'Thanks!', 'Yes!', 'Thanks!!', 'Thanks.', 'Yeah!', 'Probably...', 'Thanks!!', 'Thanks.', 'thanks', 'Roger.', 'Thanks!', 'Done.', 'Nope...', 'Thanks!', 'Yes!', 'Nope.', 'Yeah!', 'Probably...', 'Ughh...', 'Thankyou', 'EB3209D', 'eb3208C', 'sure', 'EB3209D', 'eb3208C', 'sure', 'EB3209D', 'eb3208C', 'sure', 'Enjoy', 'ok', 'yes', 'Yes', 'Fine.', 'OK', 'Sure', 'Approved', 'APPROVED', 'approved', 'aPPROVED', 'GOOD', 'No', '15500', 'Thanks', 'no', 'GOOD', 'aPPROVED', 'Approved', 'APPROVED', 'approved', 'yes', 'Sure', 'Yes', 'OK', 'Fine.', 'no', 'No', 'Enjoy', '15500', 'ok', 'Thanks', 'ok', 'Enjoy', 'Thanks', '15500', 'No', 'Yes', 'no', 'yes', 'Sure', 'OK', 'Yes', 'Fine.', 'no', 'No', 'ok', 'Enjoy', '15500', 'Thanks', 'GOOD', 'aPPROVED', 'Approved', 'APPROVED', 'approved', 'sailing', 'rocked', 'yeah', 'allegedly', 'in', '37176', 'cram', '7', '30-40', 'thanks.', 'ditto', '$65', 'no', 'mo', 'nice', 'sluts?', '0768', '9/21-9/23?', 'mediatrip.com', 'nope', 'shooters.', 'nevermind', 'totally', 'totally', 'nope', 'shooters.', 'nevermind', 'sailing', 'rocked', 'yeah', 'allegedly', 'in', '37176', 'cram', '7', '30-40', 'thanks.', 'ditto', '$65', 'no', 'mo', 'nice', 'sluts?', '0768', '9/21-9/23?', 'mediatrip.com', 'nope', 'shooters.', 'nevermind', 'totally', 'Done.', 'Done.', 'Done.', 'yes', '8/20', 'gdp-nng/vent', 'If-anr/ok', 'yes', '8/20', 'gdp-nng/vent', 'If-anr/ok', 'yes', '8/20', 'gdp-nng/vent', 'If-anr/ok', 'Yes.', '26510', 'Absolutely.', 'thanks', 'ok', 'Thanks.', 'Darn.', '63491', 'Thanks...', 'thanx', 'YIPPEEE', 'Thanks.', 'yep', 'Cool.', '63491', 'Darn.', 'Thanks.', 'ok', 'Thanks...', 'Absolutely.', '26510', 'Yes.', 'Cool.', 'yep', 'YIPPEEE', 'thanx', 'Thanks.', 'ok', 'Thanks.', 'YIPPEEE', 'Thanks.', 'Thanks!', 'Darn.', 'Thanks.', 'ok', 'Absolutely.', 'Thanks...', '26510', 'Yes.', 'Cool.', 'yep', 'YIPPEEE', 'thanx', 'Thanks.', '63491', 'yes', 'YES', 'yes', 'YES', 'YES', 'YES', 'yes', 'YES', 'fine', 'approved', 'fine', 'approved', 'fine', 'approved', 'YES', '8.41', 'YES', '8.41', 'ok', 'ok', 'YES', '8.41', 'Yes', 'thanks', 'Done', 'Done!', 'Changed.', 'Changed.', 'Done!', 'Done', 'thanks', 'Yes', 'Done', 'thanks', 'Yes', 'Changed.', 'Done!', 'thanks.', 'thanks.', 'thanks.', 'Thanks!', 'Thanks.', 'Thanks.', 'Thanks!', 'Thanks', 'Thanks!', 'yes', 'thanks!!!', 'OK', 'Thanks!!!', 'ok', 'Thanks', 'Thanks!', 'yes', 'thanks!!!', 'OK', 'Thanks!!!', 'ok', 'Yes.', 'Fine.', 'Approved.', 'Attend.', 'OK.', 'OK.', 'OK.', 'Thanks.', 'Thanks', 'Thanks.', 'Thanks', 'Thanks.', 'Thanks', 'Thanks', 'Thanks!', 'RIGHT!!!', 'Weasel!!', 'michaelpshannon@yahoo.com', 'Thanks!', 'RIGHT!!!', 'Weasel!!', 'michaelpshannon@yahoo.com', 'Thanks', 'Thanks', 'Thanks!', 'Thanks', 'Thanks!', 'RIGHT!!!', 'Weasel!!', 'michaelpshannon@yahoo.com', 'http://www.fortune.com/', 'http://www.fortune.com/', 'kruscit', 'kruscit', 'http://www.fortune.com/', 'http://www.fortune.com/', 'http://www.fortune.com/', 'kruscit', 'kruscit', 'http://www.fortune.com/', 'Yippee', 'done', 'yes', 'Hooray', 'hooray', 'Absolutely', 'sure', 'fine', 'houston', 'THANKS', 'great', 'Thanks.', 'thanks', 'great', 'confirmed', 'thanks', 'Thanks.', 'thanks', 'confirmed', 'THANKS', 'thanks', 'done', 'yes', 'Hooray', 'hooray', 'Absolutely', 'sure', 'fine', 'houston', 'THANKS', 'great', 'thanks', 'confirmed', 'thanks', 'great', 'Yippee', 'thanks', 'yes', 'I', 'ok', 'nyet', 'Non.', 'Either', 'Perfect', 'no', 'Done', '$575k', 'Your', '646-3393', 'We', 'yes', 'ok', 'nyet', 'Non.', 'Either', 'Perfect', 'no', 'Done', '$575k', '646-3393', 'Thanks', 'Thanks!', 'Thanks', 'Thanks!', 'Thanks!', 'Thanks', 'inagoodmoodarewe?', 'oh.................my...............god..............', 'inagoodmoodarewe?', 'inagoodmoodarewe?', 'oh.................my...............god..............', '28', 'closed', 'ok', 'great', 'thanks', 'agreed', 'closed', 'great', 'thanks', 'agreed', 'ok', '28', 'ok', '28', 'ok', 'closed', '28', 'great', 'thanks', 'agreed', 'A-eet', 'Helloooooooooooooooooooooooooooooooo.................................?', 'no.', 'Thanks.', 'huh?', 'OK', 'word.', 'A-eet', 'word.', 'OK', 'huh?', 'Thanks.', 'no.', 'Helloooooooooooooooooooooooooooooooo.................................?', 'A-eet', 'word.', 'OK', 'huh?', 'Thanks.', 'no.', 'Helloooooooooooooooooooooooooooooooo.................................?', 'yes', 'yes', 'yes', '\"one\"?', 'Done.', 'Thanks!', 'Thanks!', 'Done.', 'Done.', 'Thanks!', '713-853-7459', 'CONGRATULATIONS!!!!!!!!!!!', 'Approved.', 'guil15@hotmail.com', 'CONGRATULATIONS!!!!!!!!!!!', '713-853-7459', 'Approved.', 'NICE', 'EB3209C', 'eb3209c', 'EB3209C', 'eb3209c', 'NICE', 'NICE', 'NICE', 'EB3209C', 'eb3209c', 'received.', 'thanks', 'thanks!', 'Thanks!', 'Saw', 'Dwivedi', 'thanks!', 'thanks', 'Thanks!', 'Dwivedi', 'Saw', 'yes', 'yes', 'yes', 'Thanks']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pprint\n",
    "\n",
    "re_emails[re_emails['n_words'] == 1]['content'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "12144be7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>150407</th>\n",
       "      <td>&lt;32465220.1075860470133.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Energy Issues</td>\n",
       "      <td>Mary Hain has resigned her position with Enron...</td>\n",
       "      <td>27651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151120</th>\n",
       "      <td>&lt;327046.1075860343569.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Energy Issues</td>\n",
       "      <td>Mary Hain has resigned her position with Enron...</td>\n",
       "      <td>27651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152774</th>\n",
       "      <td>&lt;21414803.1075860416709.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Energy Issues</td>\n",
       "      <td>Mary Hain has resigned her position with Enron...</td>\n",
       "      <td>27651</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id            subject  \\\n",
       "150407  <32465220.1075860470133.JavaMail.evans@thyme>  Re: Energy Issues   \n",
       "151120    <327046.1075860343569.JavaMail.evans@thyme>  Re: Energy Issues   \n",
       "152774  <21414803.1075860416709.JavaMail.evans@thyme>  Re: Energy Issues   \n",
       "\n",
       "                                                  content  n_words  \n",
       "150407  Mary Hain has resigned her position with Enron...    27651  \n",
       "151120  Mary Hain has resigned her position with Enron...    27651  \n",
       "152774  Mary Hain has resigned her position with Enron...    27651  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check replied emails with max words 27651\n",
    "\n",
    "re_emails[re_emails['n_words'] == 27651]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9b7a1ba5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mary Hain has resigned her position with Enron.  Please remove her from all=\n",
      "=20\n",
      "your mail lists.\n",
      "\n",
      "Thank you.\n",
      "\n",
      "Lysa Akin\n",
      "Gov't Affairs - Sr. Admin. Ass't.\n",
      "\n",
      "\n",
      "\n",
      "To: Ann M Schmidt/Corp/Enron@ENRON, Bryan Seyfried/LON/ECT@ECT,=20\n",
      "dcasse@whwg.com, dg27@pacbell.net, Elizabeth Linnell/NA/Enron@Enron,=20\n",
      "filuntz@aol.com, James D Steffes/NA/Enron@Enron, Janet=20\n",
      "Butler/ET&S/Enron@ENRON, Jeannie Mandelker/HOU/ECT@ECT, Jeff=20\n",
      "Dasovich/NA/Enron@Enron, Joe Hartsoe/Corp/Enron@ENRON, John=20\n",
      "Neslage/ENRON_DEVELOPMENT@ENRON_DEVELOPMENT, John Sherriff/LON/ECT@ECT,=20\n",
      "Joseph Alamo/NA/Enron@Enron, Karen Denne/Corp/Enron@ENRON, Lysa=20\n",
      "Akin/PDX/ECT@ECT, Margaret Carson/Corp/Enron@ENRON, Mark=20\n",
      "Palmer/Corp/Enron@ENRON, Mark Schroeder/LON/ECT@ECT, Markus=20\n",
      "Fiala/LON/ECT@ECT, Mary Hain/HOU/ECT@ECT, Michael R Brown/LON/ECT@ECT, Mike=\n",
      "=20\n",
      "Dahlke/ENRON_DEVELOPMENT@ENRON_DEVELOPMENT, Mona L Petrochko/NA/Enron@Enron=\n",
      ",=20\n",
      "Nicholas O'Day/AP/Enron@Enron, Peggy Mahoney/HOU/EES@EES, Peter=20\n",
      "Styles/LON/ECT@ECT, Richard Shapiro/NA/Enron@Enron, Rob=20\n",
      "Bradley/Corp/Enron@ENRON, Sandra McCubbin/NA/Enron@Enron, Shelley=20\n",
      "Corman/ET&S/Enron@ENRON, Stella Chan/ENRON_DEVELOPMENT@ENRON_DEVELOPMENT,=\n",
      "=20\n",
      "Steven J Kean/NA/Enron@Enron, Susan J Mara/NA/Enron@Enron, Mike=20\n",
      "Roan/ENRON@enronXgate, Alex Parsons/EU/Enron@Enron, Andrew=20\n",
      "Morrison/LON/ECT@ECT, lipsen@cisco.com, Janel Guerrero/Corp/Enron@Enron,=20\n",
      "Shirley A Hudler/HOU/ECT@ECT, Kathleen Sullivan/NA/Enron@ENRON, Tom=20\n",
      "Briggs/NA/Enron@Enron, Linda Robertson/NA/Enron@ENRON, Lora=20\n",
      "Sullivan/Corp/Enron@ENRON, Jennifer Thome/NA/Enron@Enron,=20\n",
      "jkradin@marathon-com.com, rlichtenstein@marathon-com.com,=20\n",
      "syamane@marathon-com.com, ken@kdscommunications.com, hgovenar@govadv.com,=\n",
      "=20\n",
      "sgovenar@govadv.com, bhansen@lhom.com\n",
      "cc: =20\n",
      "Subject: Energy Issues\n",
      "\n",
      "Please see the following articles:\n",
      "\n",
      "Sac Bee, Wed, 3/28:  \"State backs rate hike: But increase still may not=20\n",
      "resolve crisis\"\n",
      "\n",
      "Sac Bee, Wed, 3/28:  \"Batten down: Electric storm brews \"\n",
      "\n",
      "Sac Bee, Wed, 3/28:  \"Use more energy, pay higher rates \"\n",
      "\n",
      "San Diego Union, Wed, 3/28:  \"Regulators prescribe biggest rate increase in=\n",
      "=20\n",
      "state history\"\n",
      "\n",
      "San Diego Union, Wed, 3/28:  \"City leaders in East County schedule forum on=\n",
      "=20\n",
      "energy woes\"\n",
      "\n",
      "San Diego Union, Wed, 3/28:  \"Developments in California's power crisis\"\n",
      "\n",
      "San Diego Union, Wed, 3/28:  \"Congressman eyes nuclear vessels as source of=\n",
      "=20\n",
      "power\"\n",
      "\n",
      "San Diego Union, Wed, 3/28:  \"Bush calls California energy crisis 'ominous'=\n",
      " \"\n",
      "\n",
      "San Diego Union, Wed, 3/28:  \"Rate hike means new political heat for=20\n",
      "California governor\"\n",
      "\n",
      "LA Times, Wed, 3/28:  \"PUC Approves Largest Electricity Rate Increase in=20\n",
      "State's History\"\n",
      "\n",
      "LA Times, Wed, 3/28:  \"With Energy Crisis Far From Over, Experts Say More=\n",
      "=20\n",
      "Hikes Possible \"\n",
      "\n",
      "LA Times, Wed, 3/28:  \"Davis Keeps His Distance From Utility Rate Hikes\"\n",
      "\n",
      "LA Times, Wed, 3/28:  \"Federal Energy Agency Unlikely to Order Refunds\"\n",
      "\n",
      "LA Times, Wed, 3/28:  \"A Painful Step\"   (Commentary)\n",
      "\n",
      "SF Chron, Wed, 3/28:  \"PUC Votes To Jack Up Power Rates=20\n",
      "Tiered increases approved as protests disrupt meeting\"\n",
      "\n",
      "SF Chron, Wed, 3/28:  \"Developments in California's power crisis\" =20\n",
      "\n",
      "SF Chron, Wed, 3/28:  \"Energy Nominees Would Give GOP Majority on Panel\"\n",
      "\n",
      "SF Chron, Wed, 3/28: \"Megawatt Foolish=20\n",
      "Strike a Pose for Energy Conservation\"\n",
      "\n",
      "SF Chron, Wed, 3/28:  \"New York mayor calls for cap on power prices \"=20\n",
      "\n",
      "SF Chron, Wed, 3/28:  \"Power regulators approve rate hikes of up to 46=20\n",
      "percent \"\n",
      "\n",
      "Mercury News, Tues, 3/27:  \"No explanation given after officials approve 27=\n",
      "=20\n",
      "percent rate hike \"\n",
      "\n",
      "Mercury News, Tues, 3/27:  \"Impact of increases will vary greatly\"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"PUC raises power rates\"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"Energy payments not enough, gas-using generator=\n",
      "s=20\n",
      "say\"\n",
      "Orange County, Wed, 3/28:  \"Bush to nominate 2 who favor free markets for=\n",
      "=20\n",
      "energy panel\"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"Davis backs power rate hikes if 'necessary' \"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"Cheney says state's on its own\"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"Highest rate hike in state history raises=20\n",
      "consumer anger\"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"Lighten up...it's just a California thing\"\n",
      "\n",
      "Orange County, Wed, 3/28:  \"Conserve and Save\"\n",
      "\n",
      "NY Times, Wed, 3/28:  \"RECORD RATE HIKE SET IN CALIFORNIA\"\n",
      "\n",
      "Individual.com, Wed, 3/28:  \"Calif. Regulators Raise Power Rates \"\n",
      "\n",
      "=20\n",
      "\n",
      "\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------------------------------------------------------\n",
      "\n",
      "State backs rate hike: But increase still may not resolve crisis\n",
      "By Carrie Peyton and Dale Kasler\n",
      "Bee Staff Writers\n",
      "(Published March 28, 2001)=20\n",
      "SAN FRANCISCO - Brushing aside the shouts and hisses of outraged protesters=\n",
      ",=20\n",
      "state regulators approved a $4.8 billion-a-year utility rate hike Tuesday b=\n",
      "ut=20\n",
      "couldn't guarantee that the increases will remedy California's electricity=\n",
      "=20\n",
      "crisis.=20\n",
      "With activists chanting \"rate hikes no way\" and utility executives branding=\n",
      "=20\n",
      "the decision as inadequate, the Public Utilities Commission voted 5-0 to=20\n",
      "raise rates about 30 percent for customers of Pacific Gas and Electric Co.=\n",
      "=20\n",
      "and Southern California Edison. The rate hike took effect immediately but=\n",
      "=20\n",
      "probably won't show up in customer bills until May.=20\n",
      "But while PUC commissioners said they were taking a bold step, major holes=\n",
      "=20\n",
      "remain in the state's patchwork effort to save the two beleaguered utilitie=\n",
      "s=20\n",
      "and ensure reliable energy supplies during what promises to be a long, hot=\n",
      "=20\n",
      "summer:=20\n",
      "The higher rates won't pay off billions in old utility debts, leaving PG&E=\n",
      "=20\n",
      "and Edison under the continued threat of a bankruptcy filing by power=20\n",
      "generators, bondholders or other creditors.=20\n",
      "The increases will likely curtail energy consumption, but it's unlikely the=\n",
      "=20\n",
      "cutbacks will be enough to stave off rolling blackouts this summer. The PUC=\n",
      "=20\n",
      "put off a crucial decision on how to allocate the rate hikes across custome=\n",
      "r=20\n",
      "categories, and the final rate design could determine how much Californians=\n",
      "=20\n",
      "will conserve.=20\n",
      "The vote may not achieve one of its key goals: reviving the wind farms,=20\n",
      "cogenerators and other=20\n",
      "alternative-energy producers that provide more than 20 percent of=20\n",
      "California's electricity. Hundreds have shut down in recent weeks because o=\n",
      "f=20\n",
      "nonpayment by PG&E and Edison, contributing to rolling blackouts March 19 a=\n",
      "nd=20\n",
      "20.=20\n",
      "After months of prodding, the PUC approved what may well be the largest=20\n",
      "utility rate increase in California's history, saying it had no choice if i=\n",
      "t=20\n",
      "was to shore up utility finances and lessen the risk of blackouts. It raise=\n",
      "d=20\n",
      "rates just under 30 percent in addition to making permanent a 9 percent=20\n",
      "surcharge approved in January.=20\n",
      "The increased revenue will go to pay for the state Department of Water=20\n",
      "Resources' power purchases on the utilities' behalf, and to=20\n",
      "alternative-energy producers that supply a major chunk of the utilities'=20\n",
      "power.=20\n",
      "The PUC's move could make utility bankruptcies more palatable to state=20\n",
      "officials because it ensures payment for future power supplies. \"Now, if we=\n",
      "=20\n",
      "have a bankruptcy, a lot of bad things will probably happen, but there's le=\n",
      "ss=20\n",
      "risk to the general public of mass power disruptions,\" said state Sen. Debr=\n",
      "a=20\n",
      "Bowen, D-Marina del Rey, chairwoman of the Senate Energy Committee.=20\n",
      "PG&E, arguing that the PUC didn't go nearly far enough, said it will demand=\n",
      "=20\n",
      "the panel reconsider.=20\n",
      "\"Today's =01( actions, while providing a welcome dose of realism in the ene=\n",
      "rgy=20\n",
      "policy debate, unfortunately do not resolve any of the key issues facing=20\n",
      "California's policymakers,\" PG&E President Gordon Smith said.=20\n",
      "Smith said the decision could worsen California's energy problems by forcin=\n",
      "g=20\n",
      "the utilities to pay the state Department of Water Resources, which is buyi=\n",
      "ng=20\n",
      "power on the utilities' behalf, ahead of other creditors.=20\n",
      "An Edison vice president, Bruce Foster, said the rate increase \"is a step i=\n",
      "n=20\n",
      "the right direction\" but added, \"Only time will tell if it's enough.\" The=\n",
      "=20\n",
      "company announced it will start making interest payments on defaulted bond=\n",
      "=20\n",
      "and bank debts.=20\n",
      "Edison and PG&E both criticized the commission for sharply lowering the PUC=\n",
      "'s=20\n",
      "estimates of the utilities' debts.=20\n",
      "PUC commissioners acknowledged they have more work to do to resolve=20\n",
      "California's energy mess - and some hinted that additional rate hikes may b=\n",
      "e=20\n",
      "in the offing.=20\n",
      "\"Have we gotten the number right? I don't know,\" said Commissioner Henry=20\n",
      "Duque. But PUC President Loretta Lynch said the $4.8 billion will be enough=\n",
      "=20\n",
      "to pay California's ongoing power bills.=20\n",
      "Meanwhile, Gov. Gray Davis repeated his opposition to rate hikes.=20\n",
      "\"We do not have all the appropriate financial numbers necessary to make a=\n",
      "=20\n",
      "decision,\" Davis said. \"Until we do, I cannot support any rate increase. =\n",
      "=01(=20\n",
      "While I have opposed rate increases, if it becomes clear that a rate increa=\n",
      "se=20\n",
      "is absolutely necessary for the good of the state, I will support one that =\n",
      "is=20\n",
      "fair and do my duty to convince Californians of its necessity.\"=20\n",
      "Yet most observers believe the PUC acted with Davis' approval - three of it=\n",
      "s=20\n",
      "members are Davis appointees - and last Friday his aides shared data with=\n",
      "=20\n",
      "business lobbyists indicating rates would rise substantially, said Jack=20\n",
      "Stewart, president of the California Manufacturers & Technology Association=\n",
      ".=20\n",
      "Consumer advocates blasted rate increases as a surrender to the power=20\n",
      "generators that control much of California's electricity.=20\n",
      "Noting that the state's grid operator has accused the generators of=20\n",
      "overcharging Californians by $6.2 billion, they urged the PUC to focus its=\n",
      "=20\n",
      "firepower on those \"robber barons on steroids,\" as Berkeley activist Barbar=\n",
      "a=20\n",
      "George put it.=20\n",
      "\"It's like dealing with the mob,\" said Doug Heller of the Foundation for=20\n",
      "Taxpayer and Consumer Rights. \"Instead of breaking our fingers, they're goi=\n",
      "ng=20\n",
      "to turn the lights out again.\"=20\n",
      "Jamming the PUC's auditorium to near capacity, activists said the state=20\n",
      "should seize the generators' power plants. They razzed the commissioners wi=\n",
      "th=20\n",
      "taunts of \"blood money!\" and \"it's a sham,\" prompting guards to escort some=\n",
      "=20\n",
      "protesters out.=20\n",
      "PUC President Lynch partially agreed with the protesters, saying the=20\n",
      "generators are \"the real culprits in our energy crisis.\" But she said the=\n",
      "=20\n",
      "commission had to approve rate hikes.=20\n",
      "The PUC ordered the utilities to help state officials fight the power=20\n",
      "generators to refund some of their profits. Lynch said the rate hikes would=\n",
      "=20\n",
      "be refunded to customers if generators are forced to surrender money.=20\n",
      "The higher rates will generate $2.51 billion in additional yearly revenue f=\n",
      "or=20\n",
      "Edison and $2.28 billion for PG&E, according to PUC estimates.=20\n",
      "But with most of the money going to reimburse the state Department of Water=\n",
      "=20\n",
      "Resources for current and future power purchases, the PUC action doesn't=20\n",
      "repay the billions the utilities owe to generators and other creditors.=20\n",
      "\"You still have $14 billion, maybe more, of costs that haven't been paid fo=\n",
      "r,=20\n",
      "and that's still a problem,\" said Gary Ackerman of the Western Power Tradin=\n",
      "g=20\n",
      "Forum, an association of power generators. \"I don't believe creditors are=\n",
      "=20\n",
      "going to be infinitely patient.\"=20\n",
      "The two utilities have defaulted on billions of debts and face a daily thre=\n",
      "at=20\n",
      "of bankruptcy.=20\n",
      "\"That's still an issue that ultimately has to be addressed,\" said Senior Vi=\n",
      "ce=20\n",
      "President John Stout of Reliant Energy Inc., a generator that's owed $370=\n",
      "=20\n",
      "million.=20\n",
      "Davis is negotiating to buy the transmission grid from Edison, PG&E and San=\n",
      "=20\n",
      "Diego Gas & Electric Co. as a means of paying off the utilities' existing=\n",
      "=20\n",
      "debts. But Bowen, the Energy Committee chair, said she doubts the deal will=\n",
      "=20\n",
      "go through.=20\n",
      "Beyond complaining the PUC didn't solve the existing-debt issue, the=20\n",
      "utilities were dismayed by a change in PUC accounting that sharply lowers=\n",
      "=20\n",
      "estimates of their debts and could have major impacts on how much control=\n",
      "=20\n",
      "regulators keep over their operations for months to come. Even though rates=\n",
      "=20\n",
      "keep going up, a state ordered rate freeze remains technically in effect, a=\n",
      "nd=20\n",
      "that freeze gives regulators ammunition to argue that utilities can't pass=\n",
      "=20\n",
      "all their costs on to customers.=20\n",
      "PG&E's Smith called it the \"most egregious\" commission action. Edison calle=\n",
      "d=20\n",
      "it \"illegal.\"=20\n",
      "The board voted to raise rates 3 cents a kilowatt-hour - and make permanent=\n",
      " a=20\n",
      "1-cent increase approved in January. Documents from Lynch's office said rat=\n",
      "es=20\n",
      "would rise on average 29 percent, but Lynch later described it as 26 percen=\n",
      "t.=20\n",
      "The PUC put off a decision on how the higher rates will affect each custome=\n",
      "r=20\n",
      "category, other than to say it's leaning toward a system that will charge=\n",
      "=20\n",
      "more for electricity consumed above a certain threshold. The PUC estimated=\n",
      "=20\n",
      "that 45 percent of residential ratepayers - those who use less than that=20\n",
      "threshold - will see no rise at all. Lynch proposed some businesses would s=\n",
      "ee=20\n",
      "rate hikes of as much as 87 percent.=20\n",
      "\"How much conservation will this get us? It will depend very much on how it=\n",
      "'s=20\n",
      "designed,\" said Severin Borenstein of the University of California Energy=\n",
      "=20\n",
      "Institute.=20\n",
      "The rate hikes won't affect customers of municipal utilities such as=20\n",
      "Sacramento Municipal Utility District. SMUD is planning a 16 percent rate=\n",
      "=20\n",
      "hike.=20\n",
      "The PUC vote may do little to help the hundreds of alternative-energy=20\n",
      "producers known as \"qualifying facilities,\" many of which have shut down fo=\n",
      "r=20\n",
      "nonpayment.=20\n",
      "The PUC on Tuesday ordered PG&E and Edison to start paying these producers =\n",
      "in=20\n",
      "April, but some of the producers say the PUC's new pricing mechanism doesn'=\n",
      "t=20\n",
      "get them back on their feet financially. Many, following the lead of one=20\n",
      "Imperial Valley producer, may choose to sever their contracts with the two=\n",
      "=20\n",
      "utilities and sell their power elsewhere, even out of state.=20\n",
      "\"This is not a solution,\" cogenerators' lawyer Jerry Bloom said of the PUC=\n",
      "=20\n",
      "vote. \"It will drive additional (plants) off the system.\"=20\n",
      "California can ill afford to lose suppliers. On Tuesday, the Independent=20\n",
      "System Operator, which runs the power grid, declared a Stage 2 alert, meani=\n",
      "ng=20\n",
      "reserve supplies had dipped to less than 5 percent of demand.=20\n",
      "And many experts believe California won't be able to avoid blackouts this=\n",
      "=20\n",
      "summer. The severe drought hitting the Pacific Northwest will deprive=20\n",
      "California of a critical source of hydroelectricity.=20\n",
      "Utility stocks, which shot up Monday in anticipation of the PUC's vote,=20\n",
      "retreated slightly on Tuesday. PG&E Corp. shares dropped 44 cents to $13.20=\n",
      ";=20\n",
      "Edison International fell 39 cents to $14.16.=20\n",
      "Bee staff writers Dan Smith, Emily Bazar and Kevin Yamamura, and Reuters ne=\n",
      "ws=20\n",
      "service contributed to this report.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------------------------------------\n",
      "Batten down: Electric storm brews=20\n",
      "By Gilbert Chan and Andrew LePage\n",
      "Bee Staff Writers\n",
      "(Published March 28, 2001)=20\n",
      "First higher gasoline prices hit Dust-Tex Service Inc. Then skyrocketing=20\n",
      "natural gas bills socked the Sacramento industrial laundry service. Now a=\n",
      "=20\n",
      "third financial blow looms - higher electricity bills.=20\n",
      "\"It's become a real nightmare. It's pretty scary. You have to find money to=\n",
      "=20\n",
      "pay these bills,\" said Sylvia Compton, co-owner of the longtime family-owne=\n",
      "d=20\n",
      "south Sacramento business.=20\n",
      "It could be worse for Compton, whose business is served by the Sacramento=\n",
      "=20\n",
      "Municipal Utility District. It will be for customers of Pacific Gas and=20\n",
      "Electric Co., who face rate hikes averaging 30 percent in addition to a 9=\n",
      "=20\n",
      "percent rate increase already approved.=20\n",
      "The California Public Utilities Commission gave PG&E and Southern Californi=\n",
      "a=20\n",
      "Edison permission to raise electric rates by $4.8 billion a year. How the=\n",
      "=20\n",
      "burden will fall on various customer groups hasn't been decided.=20\n",
      "Although SMUD customers aren't affected by the PUC's decision to sharply=20\n",
      "raise rates, they aren't likely to be immune. SMUD, also facing skyrocketin=\n",
      "g=20\n",
      "energy costs, is considering rate increases ranging from 13 percent to 25=\n",
      "=20\n",
      "percent.=20\n",
      "For Dust-Tex and scores of other California companies, any rate hike would =\n",
      "be=20\n",
      "another unwelcome increase in the cost of doing business. It will hit at a=\n",
      "=20\n",
      "time the U.S. economy is slowing and employer costs such as health benefits=\n",
      "=20\n",
      "are soaring.=20\n",
      "For consumers, it will mean a double blow, higher electricity bills at home=\n",
      "=20\n",
      "and higher prices for goods and services.=20\n",
      "Raley's, the West Sacramento-based supermarket chain, for example, uses hug=\n",
      "e=20\n",
      "amounts of electricity not only for lights and refrigeration at its stores,=\n",
      "=20\n",
      "but also to cool food at its distribution centers. Given the thin profit=20\n",
      "margins of the grocery business, it has little choice but to raise prices.=\n",
      "=20\n",
      "\"We are doing all we can at Raley's to conserve energy. But as rates go up =\n",
      "we=20\n",
      "will have to pass those increases on to our customers in the cost of goods,=\n",
      "\"=20\n",
      "said Michael Teel, Raley's president and chief executive officer.=20\n",
      "The higher electric bills approved by the PUC - aimed at keeping the state'=\n",
      "s=20\n",
      "two biggest utilities from bankruptcy - mean California businesses and=20\n",
      "consumers will have almost $5 billion less to spend each year on other=20\n",
      "things.=20\n",
      "While that amount represents a huge bite out of consumer spending, by itsel=\n",
      "f,=20\n",
      "it won't necessarily lead to a recession, said Brad Williams, senior=20\n",
      "economist at the state Legislative Analyst's Office.=20\n",
      "\"What we're really talking about is less economic growth,\" Williams said.=\n",
      "=20\n",
      "\"Some households will feel this, and it will pinch their budgets and affect=\n",
      "=20\n",
      "how much income they have for purchasing goods and services.\"=20\n",
      "The state estimates that spending on electricity last year averaged between=\n",
      " 2=20\n",
      "and 3 percent of both business production costs and household budgets.=20\n",
      "Some business owners and economists welcomed the boost in electricity rates=\n",
      ",=20\n",
      "saying in the long run, they will spur energy conservation efforts, boost=\n",
      "=20\n",
      "power generating capacity and prevent rolling blackouts and their devastati=\n",
      "ng=20\n",
      "effect on business productivity.=20\n",
      "\"It's much worse for us if we have these damn blackouts. We have 100=20\n",
      "employees, and if there's a blackout they're all sitting around twiddling=\n",
      "=20\n",
      "their thumbs and I've still got to pay them,\" said Norm Rogers, president o=\n",
      "f=20\n",
      "Z-World Inc. in Davis, which designs and builds miniature special-purpose=\n",
      "=20\n",
      "computers.=20\n",
      "Many economists and business experts believe rolling blackouts, not higher=\n",
      "=20\n",
      "electricity rates, are the biggest threat to the economy.=20\n",
      "\"It's better to have higher rates and secure power than lower rates and=20\n",
      "blackouts,\" said Stephen Levy, senior economist at the Center for Continuin=\n",
      "g=20\n",
      "Study of the California Economy. The state can work through the higher powe=\n",
      "r=20\n",
      "costs - just as it did when gasoline and diesel prices soared last year, Le=\n",
      "vy=20\n",
      "said.=20\n",
      "In reaction to the higher power rates, a small number of businesses may lea=\n",
      "ve=20\n",
      "California, expand out of state, or fail, though most will be able to weath=\n",
      "er=20\n",
      "the short run, said Robert Smiley, dean of the Graduate School of Managemen=\n",
      "t=20\n",
      "at the University of California, Davis.=20\n",
      "Most firms will give the state just six to 18 months to solve the energy=20\n",
      "crisis before \"they'll start looking elsewhere,\" Smiley said.=20\n",
      "David Molinaro, owner of Yolo Ice and Creamery Inc., a Woodland-based ice=\n",
      "=20\n",
      "cream, dairy and ice distributor, can't pick up and leave. He absorbed high=\n",
      "er=20\n",
      "diesel fuel costs for his 11 delivery trucks for six months before levying =\n",
      "a=20\n",
      "surcharge in January. There's just no way his company can eat the higher=20\n",
      "electricity costs, he said.=20\n",
      "\"It's frustrating. People are thinking you are getting rich by raising=20\n",
      "prices,\" Molinaro said.=20\n",
      "Other businesses may have little choice but to swallow the higher utility=\n",
      "=20\n",
      "costs.=20\n",
      "\"We compete with global suppliers for almost every product. It's virtually=\n",
      "=20\n",
      "impossible for us to pass these costs on,\" said Jeff Boese of the Californi=\n",
      "a=20\n",
      "League of Food Processors. \"If we don't get a handle on energy costs very=\n",
      "=20\n",
      "quickly I would suspect a significant number of food processors to leave th=\n",
      "e=20\n",
      "state.\"=20\n",
      "In past months, the rocketing energy costs have forced companies to take=20\n",
      "dramatic steps to shave expenses.=20\n",
      "Dust-Tex, for example, has scrapped pay raises, raised prices 10 percent, c=\n",
      "ut=20\n",
      "hours for some workers, put a freeze on hiring and changed auto insurance=\n",
      "=20\n",
      "coverage for its truck fleet. At the same time, it invested $60,000 for a n=\n",
      "ew=20\n",
      "energy-efficient dryer.=20\n",
      "California Family Health & Fitness, which has eight health clubs in the=20\n",
      "capital region, has tabled plans for two additional gyms until it gets a=20\n",
      "better handle on future energy costs, according to co-owner Larry Gury. But=\n",
      "=20\n",
      "Gury and his partner are looking at spending $150,000 to $200,000 at each o=\n",
      "f=20\n",
      "four gyms to install solar panels for heating the pools and running the=20\n",
      "lights.=20\n",
      "Some impacts might not be seen for some time. In the health care industry,=\n",
      "=20\n",
      "for example, the energy crisis has only added to existing financial=20\n",
      "pressures, said Larry Levitt, vice president of the Henry J. Kaiser Family=\n",
      "=20\n",
      "Foundation.=20\n",
      "Pressed by higher bills, some employers may opt to trim or eliminate health=\n",
      "=20\n",
      "benefits for workers, Levitt said.=20\n",
      "Bee staff writer Lisa Rapaport contributed to this report.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------\n",
      "Use more energy, pay higher rates=20\n",
      "By Edie Lau\n",
      "Bee Science Writer\n",
      "(Published March 28, 2001)=20\n",
      "For all the talk of electricity rates going up across the state, there's a=\n",
      "=20\n",
      "way every resident of California can avoid the price hit altogether: Just u=\n",
      "se=20\n",
      "electricity sparingly.=20\n",
      "A rate increase approved by the state Public Utilities Commission on Tuesda=\n",
      "y=20\n",
      "does not affect households that use less than 130 percent of a baseline=20\n",
      "amount of power.=20\n",
      "Forty-five percent of households already stay within that limit, according =\n",
      "to=20\n",
      "PUC President Loretta Lynch. The proportion is slightly less in Pacific Gas=\n",
      "=20\n",
      "and Electric's territory - 42 percent, utility spokeswoman Jann Tabor said.=\n",
      "=20\n",
      "The baseline amount differs from region to region and from season to season=\n",
      ".=20\n",
      "The Sacramento Municipal Utility District is not affected by the PUC-approv=\n",
      "ed=20\n",
      "increase but is planning to raise rates this spring, as well. It, too,=20\n",
      "proposes to protect the thriftiest customers by raising the price only of=\n",
      "=20\n",
      "kilowatt-hours used over baseline. District rate administrator Alan Wilcox=\n",
      "=20\n",
      "said roughly half of SMUD households consume only baseline amounts of power=\n",
      ".=20\n",
      "Everyone else has the opportunity to dodge rate increases by scaling back=\n",
      "=20\n",
      "their electricity use.=20\n",
      "How difficult might that be? If you're accustomed to using a central air=20\n",
      "conditioner all summer and unwilling to turn up the thermostat, it could be=\n",
      "=20\n",
      "difficult, by Wilcox's reckoning.=20\n",
      "\"If you're running an air-conditioning load in the Valley, you could probab=\n",
      "ly=20\n",
      "expect to get into that second tier (above baseline) a little bit,\" he said=\n",
      ".=20\n",
      "\"There are a lot of people who do not use their air conditioning a lot or=\n",
      "=20\n",
      "have swamp coolers or just go without.\"=20\n",
      "In SMUD territory, the baseline is 700 kilowatt-hours per month in summer,=\n",
      "=20\n",
      "and 620 kwh in winter for most customers. (Households that do not use gas a=\n",
      "nd=20\n",
      "are powered entirely by electricity get a higher baseline allowance.)=20\n",
      "Wilcox said the baseline is based upon average consumption - at least, it w=\n",
      "as=20\n",
      "the average when the baseline was set in the early 1990s. Consumption has=\n",
      "=20\n",
      "crept up a bit since, to 752 kwh a month last year.=20\n",
      "Under SMUD's rate-increase proposal, the summertime rate would remain at=20\n",
      "8.058 cents per kilowatt-hour for the first 700 kwh. The price would rise b=\n",
      "y=20\n",
      "10 cents a kilowatt-hour for the next 300 kwh used, up to 1,000 kwh.=20\n",
      "Big energy consumers would pay more than 16 cents for each kwh used over=20\n",
      "1,000 kwh.=20\n",
      "Bottom line: The biggest users pay the most.=20\n",
      "Details of the rate increase approved by the PUC - which will affect PG&E=\n",
      "=20\n",
      "customers, as well as those served by Southern California Edison and San=20\n",
      "Diego Gas & Electric - have yet to be worked out, but the rates are expecte=\n",
      "d=20\n",
      "to be structured in a similar tiered fashion.=20\n",
      "Currently, PG&E customers year-around pay 11.4 cents per kilowatt-hour up t=\n",
      "o=20\n",
      "the baseline limit, and 13 cents for every kilowatt-hour over baseline. The=\n",
      "=20\n",
      "average baseline throughout the territory is between 300 and 400 kwh, Tabor=\n",
      "=20\n",
      "said; it varies from region to region to account for climate.=20\n",
      "Tabor said the PG&E baseline amounts to between 50 percent and 60 percent o=\n",
      "f=20\n",
      "the average electricity use per household in a given territory. That=20\n",
      "proportion, she said, was taken from a \"Lifeline\" program in effect from 19=\n",
      "76=20\n",
      "to 1984 that was meant to provide a minimum necessary quantity of electrici=\n",
      "ty=20\n",
      "and gas at a lower cost.=20\n",
      "Tabor said 31 percent of PG&E's 4,017,000 residential customers use only=20\n",
      "baseline amounts of electricity.=20\n",
      "Bee staff writer Carrie Peyton contributed to this report.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "----\n",
      "\n",
      "Regulators prescribe biggest rate increase in state history=20\n",
      "\n",
      "\n",
      "\n",
      "Davis calls ruling for PG&E, Edison premature; consumer groups push for=20\n",
      "seizure of plants\n",
      "By Ed Mendel=20\n",
      "UNION-TRIBUNE STAFF WRITER=20\n",
      "March 28, 2001=20\n",
      "SAN FRANCISCO -- State regulators yesterday approved the biggest rate=20\n",
      "increase in California history for the customers of Pacific Gas and Electri=\n",
      "c=20\n",
      "and Southern California Edison, but Gov. Gray Davis said the increase is=20\n",
      "premature.=20\n",
      "The governor, appearing to soften his long-standing opposition, said he wou=\n",
      "ld=20\n",
      "support a rate increase only if financial information shows that a rate=20\n",
      "increase is \"necessary for the good of the state.\"=20\n",
      "The Public Utilities Commission, controlled by Davis appointees, voted 5-0=\n",
      "=20\n",
      "for an immediate rate increase. A statement from Davis, who usually demands=\n",
      "=20\n",
      "that appointees reflect his policy, did not say whether he would ask the PU=\n",
      "C=20\n",
      "or the Legislature for a rollback if he concludes the increase is unneeded.=\n",
      "=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Developments in California's power crisis=20\n",
      "Congressman eyes nuclear vessels as source of power=20\n",
      "Bush calls California energy crisis 'ominous'=20\n",
      "Rate hike means new political heat for California governor=20\n",
      "?=20\n",
      "\n",
      "\n",
      "\n",
      "\"He can ask,\" said Steve Maviglio, Davis' press secretary. \"That is one of=\n",
      "=20\n",
      "many options. But now we are waiting for the numbers to plug in the data an=\n",
      "d=20\n",
      "see if a rate increase is necessary.\"=20\n",
      "The PUC has not yet acted on the request of San Diego Gas and Electric for =\n",
      "a=20\n",
      "rate increase. SDG&E has a debt of $681 million but has been paying all of=\n",
      "=20\n",
      "its bills, unlike PG&E and Edison. Those companies say they have debt of $1=\n",
      "3=20\n",
      "billion.=20\n",
      "SDG&E is in a different situation because its rates were capped by=20\n",
      "legislation in September that guarantees payment of its debt, allowing the=\n",
      "=20\n",
      "utility to borrow. The bills of SDG&E customers soared last summer when the=\n",
      "=20\n",
      "utility became the first to be deregulated.=20\n",
      "\"Since the PUC has approved rate hikes for Edison and PG&E, we would hope=\n",
      "=20\n",
      "they give an expedited review to our request for an increase of 2.3 cents p=\n",
      "er=20\n",
      "kilowatt hour,\" said Ed Van Herik, a SDG&E spokesman.=20\n",
      "Consumer groups warned that out-of-state generators, who have been ordered =\n",
      "by=20\n",
      "FERC to make some rebates for overcharging, will make even more money under=\n",
      "=20\n",
      "the rate increase. Some are urging Davis to seize California power plants=\n",
      "=20\n",
      "owned by the generators.=20\n",
      "\"These people are war criminals,\" said Mike Florio, an attorney for The=20\n",
      "Utility Reform Network in San Francisco.=20\n",
      "The PUC meeting, which occurred on a day when the state was hit by a Stage =\n",
      "2=20\n",
      "alert because of reduced imports from the Northwest, was interrupted by=20\n",
      "shouts from demonstrators who want a state-owned power system. They chanted=\n",
      ",=20\n",
      "\"Public power now, we won't pay\" and \"Rate hikes no way, make the energy=20\n",
      "companies pay.\"=20\n",
      "Doug Heller of the Foundation for Taxpayer and Consumer Rights in Santa=20\n",
      "Monica, which is threatening to put a public power initiative on the ballot=\n",
      "=20\n",
      "next year, said some attorneys think the abrupt rate increase can be=20\n",
      "challenged in court.=20\n",
      "Assemblyman Rod Pacheco, R-Riverside, announced that he will introduce=20\n",
      "legislation to repeal the rate increase. He said a decision should be made=\n",
      "=20\n",
      "not by appointees but by elected officials who are accountable to the publi=\n",
      "c.=20\n",
      "Democratic legislative leaders, who have large majorities in both houses,=\n",
      "=20\n",
      "said Monday that they have reluctantly concluded a rate increase is=20\n",
      "necessary. Davis is scheduled to meet with Assembly Democrats this afternoo=\n",
      "n.=20\n",
      "The rate increase approved for PG&E and Edison yesterday is 3 cents per=20\n",
      "kilowatt hour. The PUC president, Loretta Lynch, said media reports of the=\n",
      "=20\n",
      "increase amounting to a 40 percent rise are inaccurate.=20\n",
      "Lynch said the increase in the monthly bills of ratepayers would be about 2=\n",
      "6=20\n",
      "percent if applied across the board. She bases her figure on the total bill=\n",
      "=20\n",
      "that includes electricity, transmission, distribution and other costs.=20\n",
      "The media's characterization of the rate increase as 40 percent is based on=\n",
      "=20\n",
      "only the cost of electricity, not the total bill. The electricity costs=20\n",
      "before the increase were 7.2 cents per kilowatt hour for Edison and 6.7 cen=\n",
      "ts=20\n",
      "for PG&E. SDG&E's rate is 6.5 cents.=20\n",
      "Lynch proposed the rate increase after a report issued last week showed tha=\n",
      "t=20\n",
      "the state, which was forced to begin buying power for the utilities in=20\n",
      "mid-January, will have to spend more than expected. A state plan to lower=\n",
      "=20\n",
      "costs through long-term contracts will fall short this summer.=20\n",
      "The state has been spending about $1.5 billion a month to buy power. The PU=\n",
      "C=20\n",
      "also voted unanimously yesterday to give the state a share of the monthly=\n",
      "=20\n",
      "bill collected by utilities, which will be used to finance a bond of $10=20\n",
      "billion or more to repay the state general fund for the power purchases.=20\n",
      "The state was jolted last week by rolling blackouts that few expected befor=\n",
      "e=20\n",
      "summer, when heat drives up the demand for electricity. A number of small=\n",
      "=20\n",
      "generators that provide a quarter of the state's power are not operating=20\n",
      "because they have not been paid by utilities.=20\n",
      "The PUC voted yesterday to order Edison and PG&E to begin paying the small=\n",
      "=20\n",
      "generators under a new formula that links their payments to the price of=20\n",
      "natural gas at the Oregon border, rather than at the Southern California=20\n",
      "border with Arizona. Prices at the Arizona border have been 50 percent=20\n",
      "higher.=20\n",
      "David Fogarty of the Western States Petroleum Association, which represents=\n",
      "=20\n",
      "50 small generators, said many of the small plants in Southern California=\n",
      "=20\n",
      "will have to continue to buy their natural gas at the higher price on the=\n",
      "=20\n",
      "Arizona border.=20\n",
      "\"Many of them will find it uneconomic to continue to operate,\" Fogarty=20\n",
      "warned.=20\n",
      "Edison has not paid the small generators in the federal \"qualifying=20\n",
      "facilities\" program since November. PG&E has been paying them only about 15=\n",
      "=20\n",
      "percent.=20\n",
      "The PUC did not order the utilities to pay off the $1.5 billion they owe to=\n",
      "=20\n",
      "the small generators. Jan Smutny-Jones of the Independent Energy Producers=\n",
      "=20\n",
      "said some of the small generators will not be able to buy gas until they ca=\n",
      "n=20\n",
      "pay their debts.=20\n",
      "Many of the small generators produce power from \"renewable\" sources such as=\n",
      "=20\n",
      "solar, wind, geothermal and biomass. But most of the small generators use=\n",
      "=20\n",
      "natural gas turbines that also produce heat for commercial purposes.=20\n",
      "Whether Lynch acted on her own when she proposed the increase or was prompt=\n",
      "ed=20\n",
      "by the Davis administration is a topic of speculation. Davis aides told=20\n",
      "reporters during a background briefing yesterday that they did not learn of=\n",
      "=20\n",
      "Lynch's proposal until Sunday night.=20\n",
      "They said the governor was briefed Monday morning and that he did not urge=\n",
      "=20\n",
      "his PUC appointees to delay the increase. An Edison official said the utili=\n",
      "ty=20\n",
      "received the proposal late Monday morning and had little time to prepare fo=\n",
      "r=20\n",
      "a hasty PUC hearing on the increase that afternoon.=20\n",
      "The two commissioners appointed by former Gov. Pete Wilson, Richard Bilas a=\n",
      "nd=20\n",
      "Henry Duque, said they have been urging a rate increase. But both complaine=\n",
      "d=20\n",
      "of the short notice and the lack of time to study the complex proposals=20\n",
      "approved yesterday.=20\n",
      "Davis aides said they do not understand how Lynch arrived at an increase of=\n",
      " 3=20\n",
      "cents per kilowatt hour. The governor is waiting for estimates of state pow=\n",
      "er=20\n",
      "purchases this summer, available generation, the effect of conservation, th=\n",
      "e=20\n",
      "potential for federally ordered refunds from generators, and other data=20\n",
      "before he makes a rate decision.=20\n",
      "The Davis appointees on the commission, Geoffrey Brown and Carl Wood, prais=\n",
      "ed=20\n",
      "Lynch for showing courageous leadership.=20\n",
      "The increase approved yesterday, which utility spokesmen said was the large=\n",
      "st=20\n",
      "in their history, comes on top of a 9 percent residential increase for Edis=\n",
      "on=20\n",
      "and PG&E imposed in January and a scheduled 10 percent increase next March,=\n",
      "=20\n",
      "when a \"rate reduction\" paid for by a $6 billion bond expires.=20\n",
      "Lynch said the rate increase approved yesterday takes effect immediately bu=\n",
      "t=20\n",
      "will not show up on customer bills until May or later. The PUC is working o=\n",
      "n=20\n",
      "a plan to design the increase to encourage conservation, giving the biggest=\n",
      "=20\n",
      "increases to those that use the most electricity.=20\n",
      "About 45 percent of the utilities' residential customers will have no=20\n",
      "increase because of legislation exempting households that use 130 percent o=\n",
      "r=20\n",
      "less of the \"baseline,\" a minimum amount of electricity use that varies wit=\n",
      "h=20\n",
      "climate zones around the state.=20\n",
      "-----------------------------------------------------------------\n",
      "\n",
      "\n",
      "City leaders in East County schedule forum on energy woes=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "UNION-TRIBUNE=20\n",
      "March 28, 2001=20\n",
      "Members of the four East County city councils will hold a workshop tomorrow=\n",
      "=20\n",
      "to talk about ways to deal with the energy crisis.=20\n",
      "The meeting is scheduled to begin at 7 p.m. at the La Mesa Community Center=\n",
      ",=20\n",
      "4975 Memorial Drive, La Mesa.=20\n",
      "Council members from La Mesa, Santee, El Cajon and Lemon Grove will discuss=\n",
      "=20\n",
      "energy-saving policies and ordinances they could enact.=20\n",
      "Cities might look at having energy-efficient traffic lights installed or=20\n",
      "requiring new houses to use alternative energy sources, said La Mesa Mayor=\n",
      "=20\n",
      "Art Madrid.=20\n",
      "\"It's acknowledging that we have a problem and coming up with a solution,\"=\n",
      "=20\n",
      "Madrid said.=20\n",
      "Kurt Kammerer of the San Diego Regional Energy Office is scheduled to give =\n",
      "a=20\n",
      "presentation to the group. Members of the audience will have an opportunity=\n",
      "=20\n",
      "to make comments.=20\n",
      "------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Developments in California's power crisis=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "ASSOCIATED PRESS=20\n",
      "March 28, 2001=20\n",
      "Here is a look at developments in California's electricity crisis:=20\n",
      "WEDNESDAY:=20\n",
      "=01) The state remains free of power alerts in the early morning as power=\n",
      "=20\n",
      "reserves stay above 7 percent.=20\n",
      "=01) The Assembly resumes hearings on California's high natural gas prices.=\n",
      "=20\n",
      "=01) The Assembly is expected to try again to pass a bill that would let th=\n",
      "e=20\n",
      "state Public Utilities Commission order Southern California Edison Co. and=\n",
      "=20\n",
      "Pacific Gas & Electric Co. to pay alternative-energy plants.=20\n",
      "TUESDAY:=20\n",
      "=01) The PUC votes unanimously to approve electricity rate hikes of up to 4=\n",
      "6=20\n",
      "percent for customers of California's two largest utilities. The board vote=\n",
      "s=20\n",
      "5-0 to increase rates immediately for PG&E and Edison despite yells from=20\n",
      "protesters. The increases are the largest in California history.=20\n",
      "=01) Stock in both utilities suffer slight losses after expectations of the=\n",
      " rate=20\n",
      "hikes Monday caused shares in PG&E Corp. to surge 29 percent and Edison=20\n",
      "International's stock to gain 30 percent. Edison International closed down =\n",
      "39=20\n",
      "cents at $14.16 a share. PG&E Holding Corp., parent of Pacific Gas &=20\n",
      "Electric, closed down 55 cents at $13.20.=20\n",
      "=01) In a Tuesday afternoon conference call, Southern California Edison say=\n",
      "s it=20\n",
      "will begin paying interest to its bondholders after weeks of failure to pay=\n",
      ".=20\n",
      "Company authorities say they have not heard whether the news will boost the=\n",
      "ir=20\n",
      "credit rating and do not specify a dollar amount for monthly interest=20\n",
      "payments.=20\n",
      "=01) The Los Angeles Department of Water and Power, which is not attached t=\n",
      "o the=20\n",
      "state power grid and is not affected by deregulation, issues a statement=20\n",
      "Tuesday assuring that customer rates \"will continue to remain stable.\"=20\n",
      "=01) Standard & Poors retains California's cautionary credit rating despite=\n",
      "=20\n",
      "state regulators' decision to raise electricity rates. The state has been o=\n",
      "n=20\n",
      "a credit watch \"with negative implications\" since it began spending $45=20\n",
      "million a day to buy electricity for customers of two nearly bankrupt=20\n",
      "utilities. The crediw watch will remain in effect despite a rate increase=\n",
      "=20\n",
      "state regulators approved Tuesday.=20\n",
      "=01) President Bush says the electricity price caps sought by California's =\n",
      "Gray=20\n",
      "Davis and other Western governors would worsen the region's energy crisis=\n",
      "=20\n",
      "instead of helping cure it. In a speech to the Kalamazoo Chamber of Commerc=\n",
      "e,=20\n",
      "Bush says price controls helped cause the gasoline crisis of the 1970s, and=\n",
      "=20\n",
      "he won't make the same mistake.=20\n",
      "=01) Joseph Fichera, the governor's chief negotiator on the purchase of=20\n",
      "transmission lines, says he is exchanging drafts of proposed agreements wit=\n",
      "h=20\n",
      "Edison officials but won't say how soon a final agreement may be reached. H=\n",
      "e=20\n",
      "says negotiations are continuing with PG&E and San Diego Gas & Electric.=20\n",
      "=01) State agencies announce a March 29 workshop in Ontario for developers =\n",
      "who=20\n",
      "hope to build peaking power plants in time for this summer. The event is=20\n",
      "similar to one held two weeks ago in Sacramento. Peaking plants are used fo=\n",
      "r=20\n",
      "short periods when power supplies run low.=20\n",
      "=01) California's Independent System Operator declares a Stage 2 alert when=\n",
      "=20\n",
      "electricity reserves were approaching only 5 percent of available power. IS=\n",
      "O=20\n",
      "spokeswoman Stephanie McCorkle says the alert was triggered when 1,000=20\n",
      "megawatts was unexpectedly lost from the Pacific Northwest. WHAT'S NEXT:=20\n",
      "=01) The Davis administration continues negotiations with Edison, PG&E and =\n",
      "San=20\n",
      "Diego Gas & Electric Co. over state acquisition of their transmission lines=\n",
      ".\n",
      "THE PROBLEM:=20\n",
      "High demand, high wholesale energy costs, transmission glitches and a tight=\n",
      "=20\n",
      "supply worsened by scarce hydroelectric power in the Northwest and=20\n",
      "maintenance at aging California power plants are all factors in California'=\n",
      "s=20\n",
      "electricity crisis.=20\n",
      "Edison and PG&E say they've lost nearly $14 billion since June to high=20\n",
      "wholesale prices that the state's electricity deregulation law bars them fr=\n",
      "om=20\n",
      "passing onto ratepayers, and are close to bankruptcy.=20\n",
      "Electricity and natural gas suppliers, scared off by the two companies' poo=\n",
      "r=20\n",
      "credit ratings, are refusing to sell to them, leading the state in January =\n",
      "to=20\n",
      "start buying power for the utilities' nearly 9 million residential and=20\n",
      "business customers.=20\n",
      "------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Congressman eyes nuclear vessels as source of power=20\n",
      "\n",
      "\n",
      "\n",
      "By Toby Eckert=20\n",
      "COPLEY NEWS SERVICE=20\n",
      "March 28, 2001=20\n",
      "WASHINGTON -- As California braces for more blackouts this summer, a key=20\n",
      "House Republican has floated a plan that includes mobilizing a federal=20\n",
      "disaster agency, waiving some environmental regulations and exploring wheth=\n",
      "er=20\n",
      "nuclear reactors aboard Navy ships could be connected to the power grid.=20\n",
      "Several Democrats immediately attacked the plan for not including electrici=\n",
      "ty=20\n",
      "price controls sought by Gov. Gray Davis and other California officials. Th=\n",
      "ey=20\n",
      "also charged that it would undermine clean air laws.=20\n",
      "The 17-point plan was drafted by Rep. Joe Barton, R-Texas, chairman of the=\n",
      "=20\n",
      "House Commerce Committee's energy and air quality subcommittee. Copies were=\n",
      "=20\n",
      "obtained by Copley News Service.=20\n",
      "Samantha Jordan, a spokeswoman for Barton, confirmed that he had circulated=\n",
      "=20\n",
      "some proposals for responding to the Western power crisis among subcommitte=\n",
      "e=20\n",
      "members late last week. Some of the ideas were deleted before the list was=\n",
      "=20\n",
      "forwarded to the White House, Jordan added, but she would not say which one=\n",
      "s.=20\n",
      "Barton has presided over several hearings on the power crunch and has said=\n",
      "=20\n",
      "that he is \"more than prepared\" to draft legislation to help California. A=\n",
      "=20\n",
      "White House task force, headed by Vice President Dick Cheney, is also=20\n",
      "developing a national energy policy.=20\n",
      "President Bush has repeatedly stated his opposition to caps on wholesale=20\n",
      "power prices. He has called for a concerted effort to increase energy=20\n",
      "production.=20\n",
      "The list of ideas Barton sent to subcommittee members was aimed at boosting=\n",
      "=20\n",
      "power generation and transmission in the West, increasing conservation and=\n",
      "=20\n",
      "preparing for blackouts.=20\n",
      "Some of the production and transmission measures called for quickly expandi=\n",
      "ng=20\n",
      "a crucial power conduit in the Central Valley known as Path 15, using feder=\n",
      "al=20\n",
      "funds for all or part of the project; allowing small power generators to se=\n",
      "ll=20\n",
      "electricity to buyers other than the state's cash-strapped utilities; and=\n",
      "=20\n",
      "establishing standards for allowing businesses with on-site electricity=20\n",
      "generation to send surplus supplies to the power grid.=20\n",
      "Other steps, sure to be unpopular with environmentalists, could be taken if=\n",
      " a=20\n",
      "state declares an \"electricity emergency.\" In addition to the idea of=20\n",
      "harnessing the energy from nuclear-powered Navy ships and submarines, they=\n",
      "=20\n",
      "include temporarily waiving nitrogen oxide emission limits at power plants,=\n",
      "=20\n",
      "allowing the construction of generating plants on federal and Indian lands,=\n",
      "=20\n",
      "restarting mothballed nuclear reactors and delaying for a year a plan to=20\n",
      "divert water in California's Trinity River from power generation to=20\n",
      "environmental restoration.=20\n",
      "On the conservation side, Barton proposed allowing states to lengthen=20\n",
      "daylight savings time and requiring federal facilities to reduce energy use=\n",
      "=20\n",
      "by 10 percent.=20\n",
      "Finally, the Federal Emergency Management Agency would be directed to open =\n",
      "an=20\n",
      "office in California, conduct an educational campaign to prepare the public=\n",
      "=20\n",
      "for blackouts and have a plan ready to provide assistance during power=20\n",
      "outages.=20\n",
      "Barton has been skeptical of the idea of having the federal government limi=\n",
      "t=20\n",
      "the cost of wholesale power in the West and did not include that among his=\n",
      "=20\n",
      "proposals. Instead, he said the Federal Energy Regulatory Commission should=\n",
      "=20\n",
      "investigate whether wholesale prices are \"unjust and unreasonable\" -- a=20\n",
      "finding FERC already has made once.=20\n",
      "That drew the criticism of four California Democrats -- Reps. Henry Waxman =\n",
      "of=20\n",
      "Los Angeles, Anna Eshoo of Atherton, Jane Harman of Torrance and Lois Capps=\n",
      "=20\n",
      "of Santa Barbara -- who sit on the House Commerce Committee. They sent a=20\n",
      "letter to Barton on Friday spelling out their objections.=20\n",
      "\"He's left off the 10-ton gorilla solution, which is directing the FERC to=\n",
      "=20\n",
      ".?.?. impose a reasonable cap on the price of wholesale electricity,\" Harma=\n",
      "n=20\n",
      "said.=20\n",
      "The Democrats also said that Barton's proposals would \"undermine the Clean=\n",
      "=20\n",
      "Air Act\" and that there is \"widespread agreement among those directly=20\n",
      "involved in California's electricity system that clean air rules are not=20\n",
      "responsible for electricity shortages.\"=20\n",
      "-----------------------------------------------------------------\n",
      "\n",
      "\n",
      "Bush calls California energy crisis 'ominous'=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "But he foresees no quick relief, opposes controls\n",
      "By George E. Condon Jr.=20\n",
      "COPLEY NEWS SERVICE=20\n",
      "March 28, 2001=20\n",
      "KALAMAZOO, Mich. -- President Bush yesterday made his most extensive remark=\n",
      "s=20\n",
      "yet on the energy crisis in California, calling the situation \"ominous\" and=\n",
      "=20\n",
      "proclaiming that \"the time to act is now.\"=20\n",
      "Despite that pledge, though, he held out little hope of any speedy relief f=\n",
      "or=20\n",
      "the hard-hit state. He also restated his opposition to energy price control=\n",
      "s.=20\n",
      "The president, who has previously addressed the energy crunch only in answe=\n",
      "rs=20\n",
      "to reporters' questions, used a \"state of the economy\" speech to a business=\n",
      "=20\n",
      "group at Western Michigan University to elaborate on his still-evolving=20\n",
      "energy policy.=20\n",
      "He broke no new ground in his remarks, but seemed to be reacting to West=20\n",
      "Coast criticism that the federal government should be more aggressive in=20\n",
      "finding a solution to the rolling blackouts plaguing the nation's most=20\n",
      "populous state.=20\n",
      "\"The energy problem wasn't created overnight and we won't solve the problem=\n",
      "=20\n",
      "overnight,\" he said. \"But we will at least start down the right road so tha=\n",
      "t=20\n",
      "the shortages we face today will not recur year after year.\"=20\n",
      "He said a task force headed by Vice President Dick Cheney will come up with=\n",
      " a=20\n",
      "long-term strategy, but he ruled out any move by his administration that=20\n",
      "would remedy the crisis quickly.=20\n",
      "\"We'll not solve the energy problem by running the energy market from out o=\n",
      "f=20\n",
      "Washington, D.C.,\" he said. \"We will solve the energy problem by freeing th=\n",
      "e=20\n",
      "creativity of the American people to find new sources of energy and to=20\n",
      "develop the new technologies that use energy better, more efficiently and=\n",
      "=20\n",
      "more cleanly.\"=20\n",
      "He singled out for criticism any call that Washington cap energy prices.=20\n",
      "\"This administration does not, and will not, support energy price controls,=\n",
      "\"=20\n",
      "he said to applause from the business-oriented audience. \"Price controls do=\n",
      "=20\n",
      "not increase supply, and they do not encourage conservation. Price controls=\n",
      "=20\n",
      "contributed to the gas lines of the 1970s. And the United States will not=\n",
      "=20\n",
      "repeat the mistake again.\"=20\n",
      "He portrayed the energy crisis as a clear threat to his hopes of revitalizi=\n",
      "ng=20\n",
      "the national economy.=20\n",
      "And he signaled that he understands Californians are facing hardships.=20\n",
      "\"The lights are dimming in California. Consumers and businesses in Californ=\n",
      "ia=20\n",
      "are paying sharply higher energy bills. And as we compare our future energy=\n",
      "=20\n",
      "needs to the currently projected domestic energy supply, we see an ominous=\n",
      "=20\n",
      "growing gap,\" he said.=20\n",
      "He said the blame for the problem belongs to previous decision-makers.=20\n",
      "\"Our people are paying a high price for years of neglect. And the time to a=\n",
      "ct=20\n",
      "is now,\" he said.=20\n",
      "Meanwhile, Bush said he would nominate utility regulators from Texas and=20\n",
      "Pennsylvania to fill vacancies at the Federal Energy Regulatory Commission,=\n",
      "=20\n",
      "or FERC, the agency that has played a pivotal role in the California crisis=\n",
      ".=20\n",
      "However, Bush stopped short of replacing the chairman of the commission,=20\n",
      "Curtis Hebert, who has drawn fire from Gov. Gray Davis and other California=\n",
      "=20\n",
      "officials for his outspoken faith in open power markets and staunch=20\n",
      "opposition to federal controls on the soaring price of wholesale power sold=\n",
      "=20\n",
      "in the West.=20\n",
      "There was widespread speculation that Hebert, a Republican, was on his way=\n",
      "=20\n",
      "out as chairman.=20\n",
      "A White House spokeswoman would not say whether Bush has ruled out naming=\n",
      "=20\n",
      "another commissioner to the top job at a later date.=20\n",
      "Bush nominated Patrick Henry Wood III and Nora Mead Brownell to round out t=\n",
      "he=20\n",
      "five-member commission.=20\n",
      "Wood is chairman of the Texas Public Utility Commission and was an adviser =\n",
      "to=20\n",
      "Bush on energy issues during the presidential transition. Many observers ha=\n",
      "d=20\n",
      "expected Bush to name him chairman of FERC.=20\n",
      "Brownell is a member of the Pennsylvania Public Utility Commission.=20\n",
      "Both nominees, who face Senate confirmation, were praised by power industry=\n",
      "=20\n",
      "officials and consumer groups. They have helped steer deregulation plans in=\n",
      "=20\n",
      "their states that are thought to be on sounder footing than California's=20\n",
      "failed experiment.\n",
      "------------------------------------------------------------------\n",
      "\n",
      "\n",
      "Rate hike means new political heat for California governor=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "By Alexa Haussler\n",
      "ASSOCIATED PRESS=20\n",
      "March 28, 2001=20\n",
      "SACRAMENTO =01) Rolling blackouts and a record increase in electricity rate=\n",
      "s are=20\n",
      "putting the political squeeze on Gov. Gray Davis just as he prepares for hi=\n",
      "s=20\n",
      "re-election campaign.=20\n",
      "The Democratic governor took a hit Tuesday when the Public Utilities=20\n",
      "Commission =01) three of its five members appointed by Davis =01) approved =\n",
      "the very=20\n",
      "rate hikes he has repeatedly said would not be needed.=20\n",
      "The power crunch is just the weakness political foes are looking for; it=20\n",
      "could leave everyone from business owners to consumers looking for someone =\n",
      "to=20\n",
      "blame at the polls next year.=20\n",
      "\"He's the face of the problem right now and so he is going to be the target=\n",
      ",\"=20\n",
      "said Nancy Snow, a political scientist at the University of California, Los=\n",
      "=20\n",
      "Angeles.=20\n",
      "Davis already has been criticized as having a leadership style that is a=20\n",
      "\"little bit too much in the mushy, middle-of-the-road area,\" Snow said.=20\n",
      "\"That's something that he is going to have to overcome in order to=20\n",
      "politically survive this crisis.\"=20\n",
      "California Republicans blame the first-term governor for four widespread=20\n",
      "power outages this year that they call \"Gray-outs.\" His first major=20\n",
      "challenger in 2002, GOP Secretary of State Bill Jones, is making energy a=\n",
      "=20\n",
      "campaign centerpiece.=20\n",
      "A top state financial official and fellow Democrat, Controller Kathleen=20\n",
      "Connell, says Davis' decision to buy power for Southern California Edison a=\n",
      "nd=20\n",
      "Pacific Gas and Electric is putting the state in financial danger.=20\n",
      "And Democratic lawmakers are privately grumbling about Davis' handling of t=\n",
      "he=20\n",
      "crisis as consumer groups threaten a revolt on the 2002 ballot. With up to =\n",
      "25=20\n",
      "million Californians affected by the rate hikes, consumer activists hope to=\n",
      "=20\n",
      "attract millions of rebels to the polls.=20\n",
      "The PUC voted unanimously to immediately raise rates up to 46 percent for=\n",
      "=20\n",
      "Edison and PG&E customers, saying that would reduce power use this summer a=\n",
      "nd=20\n",
      "help keep the cash-starved utilities in business.=20\n",
      "The increases =01) which come on top of already-approved hikes of 9 to 15=\n",
      "=20\n",
      "percent and a 10 percent increase planned next year =01) are the biggest in=\n",
      "=20\n",
      "state history.=20\n",
      "Davis issued a written statement Tuesday easing his ardent opposition to a=\n",
      "=20\n",
      "rate hike.=20\n",
      "\"While I have opposed rate increases, if it becomes clear that a rate=20\n",
      "increase is absolutely necessary for the good of the state, I will support=\n",
      "=20\n",
      "one that is fair and do my duty to convince Californians of its necessity,\"=\n",
      "=20\n",
      "Davis said.=20\n",
      "Davis' aides are frantically trying to deflect criticism surrounding the=20\n",
      "increase and assure voters the governor is working to build more power plan=\n",
      "ts=20\n",
      "and boost conservation to avert blackouts during the hot summer months.=20\n",
      "Davis spokesman Steve Maviglio said the governor opposes a rate increase bu=\n",
      "t=20\n",
      "has no power to order the PUC to maintain current rates.=20\n",
      "\"They are an independent body, they can do what they want,\" Maviglio said.=\n",
      "=20\n",
      "He insisted the governor wasn't informed of PUC Chairwoman Loretta Lynch's=\n",
      "=20\n",
      "plan to announce a rate increase Monday, declaring to reporters: \"It's not =\n",
      "a=20\n",
      "story =01) it's the truth.\"=20\n",
      "Still, some Republicans and Democrats have said the governor has been=20\n",
      "inconsistent about his pull with the commission.=20\n",
      "He announced last week at a dinner with state labor leaders that he had=20\n",
      "directed the PUC to order utilities to start paying overdue bills to=20\n",
      "facilities that produce renewable energy.=20\n",
      "In addition, Davis administration officials informed several key Assembly=\n",
      "=20\n",
      "members Friday that the state's power-buying for Edison and PG&E could cost=\n",
      "=20\n",
      "far more than the $10 billion lawmakers and Davis estimated when they=20\n",
      "approved legislation authorizing the power purchases, making a rate increas=\n",
      "e=20\n",
      "of 50 percent or more necessary.=20\n",
      "\"He is fully dedicated to solving this thing, he doesn't just want to lay i=\n",
      "t=20\n",
      "on the lap of a ratepayer or a taxpayer, and he's trying to come up with wa=\n",
      "ys=20\n",
      "to do that,\" Maviglio said.=20\n",
      "The national attention the power problems draw is raising Davis' name=20\n",
      "recognition around the country =01) for better or worse =01) as he consider=\n",
      "s=20\n",
      "whether to make a presidential run in 2004.=20\n",
      "Davis has been making the rounds on the national TV news and Sunday morning=\n",
      "=20\n",
      "talk shows to explain to the nation why the lights have been going out in=\n",
      "=20\n",
      "California.=20\n",
      "He has attempted to shield himself by continually commenting that he=20\n",
      "inherited the energy problems; then-Republican Gov. Pete Wilson signed the=\n",
      "=20\n",
      "1996 utility deregulation bill largely blamed for the crisis, and Californi=\n",
      "a=20\n",
      "went years with no new power plants, Davis says.=20\n",
      "Republicans turn the criticism back at him. The state GOP has already run=\n",
      "=20\n",
      "radio ads criticizing Davis' handling of the power problems and are launchi=\n",
      "ng=20\n",
      "an all-out effort to blame him for the rate increases.=20\n",
      "\"It's a colossal failure of mismanagement on the part of the governor's PUC=\n",
      ",\"=20\n",
      "said Assemblyman Keith Richman, R-Northridge.=20\n",
      "But even with the energy crisis, Republicans have their work cut out for th=\n",
      "em=20\n",
      "in 2002.=20\n",
      "The state GOP =01) long shackled by infighting between moderates and=20\n",
      "conservatives =01) is in the minority in the Legislature, holds just one=20\n",
      "statewide office, Jones', and must come up with millions to match the nearl=\n",
      "y=20\n",
      "$26 million Davis had raised nearly two years before the election.=20\n",
      "And polls have shown residents growing increasingly frustrated with the=20\n",
      "energy crisis, but not blaming Davis. Those surveys were taken before=20\n",
      "Tuesday's rate hike and before California was hit last week with rolling=20\n",
      "blackouts from San Diego to the Oregon border.=20\n",
      "The governor's political campaign isn't taking any chances, moving quickly =\n",
      "to=20\n",
      "bite back at critics.=20\n",
      "Garry South, Davis' chief political adviser, issued a statement Tuesday=20\n",
      "saying Jones has failed to put forth his own solution.=20\n",
      "\"Put your megawatts where your mouth is,\" South said.=20\n",
      "------------------------------------------------------------------\n",
      "PUC Approves Largest Electricity Rate Increase in State's History=20\n",
      "\n",
      "Regulation: Some customers will pay up to 46% more. Commissioners, facing=\n",
      "=20\n",
      "strong opposition before their unanimous vote, say they are trying to avert=\n",
      "=20\n",
      "blackouts. Critics say move won't solve the crisis.=20\n",
      "\n",
      "By NANCY VOGEL and TIM REITERMAN, Times Staff Writers=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "PUC President Loretta Lynch and Commissioner Richard Bilas show strain of t=\n",
      "he=20\n",
      "meeting at which the panel announced a rate hike. \"We maintain the=20\n",
      "responsibility to keep the lights on,\" Lynch said\n",
      "ROBERT DURELL / Los Angeles Times\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "?????SAN FRANCISCO--With protesters jeering their disapproval, besieged=20\n",
      "utility regulators Tuesday adopted the largest electricity rate hike in the=\n",
      "=20\n",
      "state's history and defended the action as the only way to keep the lights =\n",
      "on=20\n",
      "for millions of customers.\n",
      "?????The California Public Utilities Commission unanimously approved an=20\n",
      "increase of 3 cents per kilowatt-hour. That will boost rates by as much as=\n",
      "=20\n",
      "42% for some Southern California Edison customers and up to 46% for some=20\n",
      "served by Pacific Gas & Electric Co.\n",
      "?????The rate increase, which will cost customers nearly $5 billion a year,=\n",
      "=20\n",
      "takes effect immediately but will not show up on utility bills until at lea=\n",
      "st=20\n",
      "May.\n",
      "?????The rate hike is the most far-reaching and politically volatile action=\n",
      "=20\n",
      "the state has taken thus far to ease an 11-month-old power crisis that has=\n",
      "=20\n",
      "nearly bankrupted the two largest utilities and triggered blackouts. But=20\n",
      "critics say it does not solve the fundamental problem of runaway wholesale=\n",
      "=20\n",
      "electricity prices.\n",
      "?????\"The PUC has done all it can do to fight wholesale energy prices that=\n",
      "=20\n",
      "are unjust and unreasonable,\" said commission President Loretta Lynch. \"We=\n",
      "=20\n",
      "maintain the responsibility to keep the lights on. . . . I believe that=20\n",
      "adding another 3 cents will comprehensively address the need for revenues.\"\n",
      "?????The commission took steps to have utilities begin paying the state=20\n",
      "Department of Water Resources for the nearly $3 billion in power it has=20\n",
      "purchased for them since mid-January.\n",
      "?????The PUC also ordered utilities to start paying for future power from=\n",
      "=20\n",
      "alternative energy producers who are starved for cash and struggling to=20\n",
      "operate. The blackouts last Monday and Tuesday were precipitated in part by=\n",
      "=20\n",
      "loss of power from such companies.\n",
      "?????But it was the rate increase proposal that packed the auditorium at PU=\n",
      "C=20\n",
      "headquarters. And the drama of the vote--and the vocal opposition--spurred=\n",
      "=20\n",
      "each of the five commissioners to forcefully defend the unpopular action.\n",
      "?????\"These are extraordinary moments in California's history,\" declared=20\n",
      "Commissioner Geoffrey Brown, a recent appointee of Gov. Gray Davis. \"And=20\n",
      "extraordinary moments require extraordinary courage. Loretta Lynch has take=\n",
      "n=20\n",
      "a lot of bad hits. But it was she who stepped up to the role of leader in=\n",
      "=20\n",
      "California.\"\n",
      "\n",
      "?????Rate Increase Lacked Support\n",
      "?????Lynch proposed the rate increase, even though an administrative law=20\n",
      "judge at the PUC concluded that an increase was unnecessary and the PUC's o=\n",
      "wn=20\n",
      "consumer protection arm, the Office of Ratepayer Advocates, opposed it. She=\n",
      "=20\n",
      "also did not have the public support of Davis, who appointed her and two=20\n",
      "other commissioners.\n",
      "?????Davis has distanced himself from the PUC's action, saying the commissi=\n",
      "on=20\n",
      "was acting independently and that he remains unconvinced that a rate increa=\n",
      "se=20\n",
      "is necessary.\n",
      "?????Commissioners Henry Duque and Richard Bilas--who were appointed by=20\n",
      "Republican Gov. Pete Wilson--called the rate hikes long overdue.\n",
      "?????\"Unless ratepayers want to face substantial rolling blackouts this=20\n",
      "summer, we have to start paying our power bills,\" Duque said. \"It cannot be=\n",
      "=20\n",
      "done with the current rates.\"\n",
      "?????Saying \"there is just no blood left in the turnip,\" Bilas warned that=\n",
      "=20\n",
      "\"absent an immediate rate increase, the utilities will be in Bankruptcy=20\n",
      "Court.\"\n",
      "?????Before the vote, sign-carrying protesters chanted, \"Rate hikes, no way=\n",
      ",=20\n",
      "make the energy companies pay.\" And the commission heard a parade of=20\n",
      "witnesses, most opposing the rate increase. Some called on the state to use=\n",
      "=20\n",
      "eminent domain to seize power plants, as Davis said he might do as a last=\n",
      "=20\n",
      "resort.\n",
      "?????Doug Heller, of the Foundation for Taxpayer and Consumer Rights in San=\n",
      "ta=20\n",
      "Monica, told the commission, \"Go to these power pirates and tell them the=\n",
      "=20\n",
      "state treasury is not their money orchard.\"\n",
      "?????Barbara George, of Women's Energy Matters, said, \"We should give the=\n",
      "=20\n",
      "generators 24 hours to sign their plants over to the state and leave.\" Geor=\n",
      "ge=20\n",
      "was one of several people ejected for repeatedly disrupting the meeting.\n",
      "?????At a hearing Monday, representatives of agriculture and manufacturing=\n",
      "=20\n",
      "industries warned that the increase would hurt their members and the=20\n",
      "California economy.\n",
      "?????The rate hike is the state's largest, according to the Office of=20\n",
      "Ratepayer Advocates, which researched the issue.\n",
      "?????Because the electricity rate is just one of many charges on a monthly=\n",
      "=20\n",
      "utility bill, the overall effect will be about a 28% increase in monthly=20\n",
      "bills, said PUC staff.\n",
      "?????Before the PUC action, homeowners and renters in Edison territory paid=\n",
      "=20\n",
      "7.2 cents per kilowatt-hour for electricity and another 5.3 cents per=20\n",
      "kilowatt-hour for other services. The new rate, totaling 15.5 cents per=20\n",
      "kilowatt-hour, boosts the overall bill by about 24%. The increase is about=\n",
      "=20\n",
      "26% for PG&E residential customers. But under Lynch's proposal, those=20\n",
      "increases would not be applied equally.\n",
      "?????To encourage conservation, she seeks a tiered system that would charge=\n",
      "=20\n",
      "miserly consumers of energy no more than current rates but would impose the=\n",
      "=20\n",
      "new higher rates on bigger users.\n",
      "?????The proposal aims the rate increase at those customers who use 30% or=\n",
      "=20\n",
      "more beyond a so-called baseline amount, which is about half of the average=\n",
      "=20\n",
      "residential use per month and varies by region.\n",
      "?????Exactly who will pay the higher electricity rate will be determined by=\n",
      "=20\n",
      "the PUC in hearings over the next month. Under Lynch's proposal, nearly hal=\n",
      "f=20\n",
      "of all residential customers would be spared rate increases.\n",
      "\n",
      "?????Tiered Rates to Encourage Saving\n",
      "?????Energy experts say tiered rates would encourage conservation, but they=\n",
      "=20\n",
      "disagree about how much.\n",
      "?????Davis energy advisor S. David Freeman, general manager of the Los=20\n",
      "Angeles Department of Water and Power, said utilities have long used tiered=\n",
      "=20\n",
      "rates when charging low-income customers. \"So this is not brand new. It's=\n",
      "=20\n",
      "just a more aggressive use of it,\" he said.\n",
      "?????UC Irvine economist Peter Navarro said he doubted that tiered rates=20\n",
      "would do much to stave off blackouts. The greatest conservation gains come=\n",
      "=20\n",
      "from investment in efficient appliances, he said, and that's why tiered rat=\n",
      "es=20\n",
      "are \"no magic bullet.\"\n",
      "?????The increase will not show up in utility bills until May at the=20\n",
      "earliest, said Paul Clanon, chief of the PUC's energy division.\n",
      "?????The higher charges for electricity consumed between now and whenever t=\n",
      "he=20\n",
      "new rate structure is incorporated in monthly bills will be calculated=20\n",
      "retroactively and charged to future bills, Clanon said.\n",
      "?????Utility executives and consumer advocates all threw up their hands at=\n",
      "=20\n",
      "the question of whether the rate hike will be enough.\n",
      "?????\"Have we gotten the number right?\" Duque said. \"I don't know. I hope w=\n",
      "e=20\n",
      "have. But this is a step in the right direction, and we'll return any money=\n",
      "=20\n",
      "not used to buy power.\"\n",
      "?????Bruce Foster, a Southern California Edison vice president, also said t=\n",
      "he=20\n",
      "rate increase is a step in the right direction. \"Only time will tell,\" he=\n",
      "=20\n",
      "said, \"if it is sufficient to meet the need.\"\n",
      "?????In a statement, PG&E welcomed the PUC action as a \"dose of realism,\" b=\n",
      "ut=20\n",
      "said the commission left unresolved a host of issues, including how much=20\n",
      "money the utility will have to pay for power that the state purchases for=\n",
      "=20\n",
      "PG&E customers.\n",
      "?????The PUC estimates that the increase will bring in an additional $4.8=\n",
      "=20\n",
      "billion annually from the 24 million people served by Edison and PG&E.\n",
      "?????But that amount doesn't go far in today's California wholesale=20\n",
      "electricity market.\n",
      "?????The state and utilities spent $5.2 billion in January buying=20\n",
      "electricity. The state, through the Department of Water Resources, has spen=\n",
      "t=20\n",
      "nearly $3 billion in taxpayer money to buy electricity on behalf of the=20\n",
      "utilities since Jan. 19 because power generators refused to sell to the=20\n",
      "cash-strapped utilities.\n",
      "?????Money utilities collect from their customers, even with the new rate=\n",
      "=20\n",
      "hike, is expected to cover a wide array of expenses, including payments on=\n",
      "=20\n",
      "$10 billion or more in revenue bonds that the state hopes to sell this spri=\n",
      "ng=20\n",
      "to cover the state's cost of buying power.\n",
      "?????\"The numbers don't add up,\" said Jason Zeller, an attorney with the=20\n",
      "Office of Ratepayer Advocates. \"Massive [additional rate] increases are my=\n",
      "=20\n",
      "concern. We are sticking a finger in the dike unless wholesale prices come=\n",
      "=20\n",
      "down.\"\n",
      "---=20\n",
      "?????Times staff writer Mitchell Landsberg in Los Angeles contributed to th=\n",
      "is=20\n",
      "story.\n",
      "--------------------------------------------------------------------------\n",
      "With Energy Crisis Far From Over, Experts Say More Hikes Possible=20\n",
      "\n",
      "Power: PUC action leaves questions unanswered, such as how to handle=20\n",
      "utilities' debts, whether to buy grid.=20\n",
      "\n",
      "By JENIFER WARREN and MIGUEL BUSTILLO, Times Staff Writers=20\n",
      "\n",
      "?????As painful as Tuesday's electricity rate increase will be for some=20\n",
      "consumers, power experts across the ideological spectrum say California is=\n",
      "=20\n",
      "not yet close to being free of the energy crisis that has engulfed it and=\n",
      "=20\n",
      "threatened its economy.\n",
      "?????Because of that, still more rate hikes are possible, they say.\n",
      "?????Consumer advocates, power company officials, legislators and regulator=\n",
      "s=20\n",
      "disagree widely on the causes of and cures for the state's problems. But th=\n",
      "ey=20\n",
      "agree that California's energy woes are dizzyingly complex. And the rate hi=\n",
      "ke=20\n",
      "makes nary a dent in many dimensions of the problem.\n",
      "?????\"Is it over? Heavens no!\" said Assemblywoman Jackie Goldberg (D-Los=20\n",
      "Angeles), who heads a state Assembly panel investigating summertime power=\n",
      "=20\n",
      "supply issues. \"It's a necessary step, but we're certainly not out of the=\n",
      "=20\n",
      "woods yet.\"\n",
      "?????The increase, along with another important action Tuesday by the PUC,=\n",
      "=20\n",
      "did resolve two major issues--how the state will continue to buy electricit=\n",
      "y=20\n",
      "that the financially shaky utility companies cannot purchase and how=20\n",
      "electricity consumers will repay California's treasury for those purchases.\n",
      "?????But the commission left unresolved a host of other questions: what to =\n",
      "do=20\n",
      "with the roughly $14 billion in debt the state's two biggest utility=20\n",
      "companies racked up in recent months buying expensive power; whether the=20\n",
      "state will buy the electricity transmission grid those utilities own; where=\n",
      "=20\n",
      "to find about $1.5 billion the utilities owe to producers of alternative=20\n",
      "energy, many of whom have not been paid in months; and how to wrap up=20\n",
      "negotiations to sign long-term contracts with generators to supply power in=\n",
      "=20\n",
      "years to come.\n",
      "?????Said PUC President Loretta Lynch: \"Until we can get wholesale prices=\n",
      "=20\n",
      "under control and get folks to conserve, we're going to be in this pickle f=\n",
      "or=20\n",
      "a while.\"\n",
      "?????Resolving the issue of repaying the treasury for power purchases was=\n",
      "=20\n",
      "crucial because the state is spending $45 million to $55 million a day to b=\n",
      "uy=20\n",
      "electricity. To cover those costs, the state is supposed to float bonds,=20\n",
      "which would be repaid from electricity bills. But without the higher rates,=\n",
      "=20\n",
      "officials feared there would be no way to cover the bond payments. And=20\n",
      "without the bonds, all state taxpayers--and not just customers of the priva=\n",
      "te=20\n",
      "utilities--would be on the hook for the debt.\n",
      "?????In addition to raising rates, the PUC on Tuesday approved a formula th=\n",
      "at=20\n",
      "will determine the size of the bond issue, expected to be the largest of it=\n",
      "s=20\n",
      "kind in U.S. history.\n",
      "?????The formula allows for $12 billion in bonds. The state's three major=\n",
      "=20\n",
      "private utilities, Southern California Edison, Pacific Gas & Electric and S=\n",
      "an=20\n",
      "Diego Gas & Electric, will have to determine how much power the state is=20\n",
      "buying for them--the figure is roughly 30%--and use a proportionate amount =\n",
      "of=20\n",
      "the money they collect from customers to pay the state back.\n",
      "?????A $12-billion bond issue is significantly larger than earlier estimate=\n",
      "s=20\n",
      "that pegged it at $10 billion. But even the larger number is far less than=\n",
      "=20\n",
      "some believe will eventually be needed to cover the cost of power the state=\n",
      "=20\n",
      "is buying.\n",
      "?????Unless prices are stabilized, state Treasurer Phil Angelides and other=\n",
      "s=20\n",
      "have warned, further bond issues may be needed. Those, in turn, could requi=\n",
      "re=20\n",
      "more rate increases.\n",
      "?????\"It's possible that, with the rate increase and if we do a good job wi=\n",
      "th=20\n",
      "conservation, it's possible that [the bonds] may be less,\" said state Sen.=\n",
      "=20\n",
      "Debra Bowen (D-Marina del Ray). \"It's also possible that, given the situati=\n",
      "on=20\n",
      "this summer, where we have not been able to secure anything close to our=20\n",
      "power needs . . . we could be paying significantly more.\"\n",
      "?????The rate increase, which kicks in only for customers who use=20\n",
      "significantly more power than a \"baseline\" amount, is intended to ease the=\n",
      "=20\n",
      "situation by inducing conservation and thereby cutting demand--forcing=20\n",
      "\"energy hogs\" to switch off spa pumps or pay the price, as Lynch put it.\n",
      "?????But no one can predict how the \"hogs\" will respond or whether the rate=\n",
      "=20\n",
      "hike--plus conservation incentives moving through the Legislature and the=\n",
      "=20\n",
      "completion of several new power plants this summer--will be enough to spare=\n",
      "=20\n",
      "the state from forced outages.\n",
      "?????Tuesday did little to fuel optimism. Despite temperatures far milder=\n",
      "=20\n",
      "than on a typical summer day, another Stage 2 energy alert was declared by=\n",
      "=20\n",
      "the state's Independent System Operator after imports dropped sharply. The=\n",
      "=20\n",
      "state avoided blackouts, \"but it was really close,\" a Cal-ISO spokeswoman=\n",
      "=20\n",
      "said.\n",
      "?????Some state forecasts have California squeaking through the hot months=\n",
      "=20\n",
      "with no outages, but private forecasters see at least 20 hours of blackouts=\n",
      ".=20\n",
      "And Reliant Energy, one of the power plant owners, predicts 1,100 hours of=\n",
      "=20\n",
      "blackouts in the state this year.\n",
      "?????Beyond the problems of supply, the state has still not resolved the=20\n",
      "question of the utilities' debt. Edison and PG&E say they've lost nearly $1=\n",
      "4=20\n",
      "billion since June because they have had to pay high wholesale power prices=\n",
      "=20\n",
      "that the state's deregulation law bars them from passing on to ratepayers.\n",
      "?????Although the PUC estimates that the rate increase will generate an=20\n",
      "additional $4.8 billion in revenue annually, most of that will go to paying=\n",
      "=20\n",
      "off bonds and other expenses. None is earmarked to relieve the utilities'=\n",
      "=20\n",
      "debt.\n",
      "?????\"When you do the math, the amount of money just doesn't get you very=\n",
      "=20\n",
      "far,\" said Assemblyman Fred Keeley (D-Boulder Creek).\n",
      "?????That is where buying the transmission lines is supposed to enter the=\n",
      "=20\n",
      "picture. Gov. Gray Davis wants the state to pay $7 billion for the lines.=\n",
      "=20\n",
      "PG&E and Edison could then use the infusion of cash to pay off their bills=\n",
      "=20\n",
      "and become credit-worthy once more.\n",
      "?????But negotiations over sale of the transmission grid seem to have bogge=\n",
      "d=20\n",
      "down. Meanwhile, support for the deal in the Legislature appears to be=20\n",
      "slipping. Republicans have long been opposed, but now support among Democra=\n",
      "ts=20\n",
      "is waning.\n",
      "?????Keeley, a critic of the transmission purchase all along, believes that=\n",
      "=20\n",
      "the state should instead buy the utilities' hydro-electric generators,=20\n",
      "contending that they would bring in revenue from the power generated at the=\n",
      "=20\n",
      "dams.\n",
      "?????\"We're just going to sink dough into that [transmission system],\" Keel=\n",
      "ey=20\n",
      "said, adding that the grid needs an estimated $1 billion in repairs. The=20\n",
      "hydro system, by contrast, would be a revenue-producing asset that could he=\n",
      "lp=20\n",
      "lower rates, he said.\n",
      "?????A related question--still unresolved--is the fate of the state's nearl=\n",
      "y=20\n",
      "700 producers of alternative and renewable energy, which supply more than a=\n",
      "=20\n",
      "quarter of the electricity used by California consumers. In a separate acti=\n",
      "on=20\n",
      "Tuesday, the PUC ordered utilities to begin paying the producers--whose=20\n",
      "megawatts are desperately needed and who, in many instances, have not been=\n",
      "=20\n",
      "paid since November.\n",
      "?????On Tuesday, the small producers delivered a little more than half of=\n",
      "=20\n",
      "their usual supplies to the utilities--a trend that has dragged on for=20\n",
      "several weeks and last week helped nudge the state into blackouts.\n",
      "?????Producers welcomed the PUC's order that they start getting paid. But,=\n",
      "=20\n",
      "they said, the move failed to address back payments, estimated at $1.5=20\n",
      "billion, owed by Edison and PG&E.\n",
      "?????\"Generators find it more predictable and less risky to operate in Thir=\n",
      "d=20\n",
      "World countries than they do in the state of California,\" griped Marty Quin=\n",
      "n,=20\n",
      "executive vice president and chief operating officer of Ridgewood Power LLC=\n",
      ",=20\n",
      "which owns three small gas-fired generators.\n",
      "?????Any move to resolve that issue, of course, would increase the debt owe=\n",
      "d=20\n",
      "by the utilities.\n",
      "?????On another front, state negotiators are finding it difficult to seal=\n",
      "=20\n",
      "long-term electricity contracts at rates more favorable than those on the=\n",
      "=20\n",
      "volatile spot market.\n",
      "?????So far, only 22 contracts have been signed, in part because generators=\n",
      "=20\n",
      "are reluctant to commit to lower prices while they're still owed billions o=\n",
      "f=20\n",
      "dollars by the near-bankrupt utilities. Experts say the power secured by th=\n",
      "e=20\n",
      "current contracts will supply less than half of what the state will need th=\n",
      "is=20\n",
      "summer.\n",
      "?????Despite all the unsettled questions, at least one utility analyst, Dan=\n",
      "=20\n",
      "Ford of the Lehman Bros. investment firm, said the PUC's actions should=20\n",
      "remove the immediate threat of utility bankruptcy and begin to lift the=20\n",
      "\"California cloud\" over electricity generators.\n",
      "?????But in a sign that the energy crisis is still causing concern on Wall=\n",
      "=20\n",
      "Street, Standard & Poor's said it would continue to carry a cautionary cred=\n",
      "it=20\n",
      "rating for the state.\n",
      "?????The rating agency, which placed California on a \"credit watch\" after t=\n",
      "he=20\n",
      "state government began spending millions to purchase electricity, said in a=\n",
      "=20\n",
      "news release that it was not clear whether the rate hike would be enough.\n",
      "?????Hovering in the background, meanwhile, is the threat of a ballot=20\n",
      "initiative that could unravel whatever solutions state officials ultimately=\n",
      "=20\n",
      "craft. Leaders of the Foundation for Taxpayer and Consumer Rights have vowe=\n",
      "d=20\n",
      "that any utility \"bailout\" would likely trigger a \"voter revolt.\"\n",
      "?????\"The rate increase certainly makes an initiative more likely,\" the=20\n",
      "foundation's lobbyist, Doug Heller, said Tuesday. If Gov. Davis does not=20\n",
      "protect the public from \"profiteers,\" Heller said, his group is ready to pu=\n",
      "sh=20\n",
      "forward with an initiative in 2002.\n",
      "---=20\n",
      "?????Times staff writers Virginia Ellis, Dan Morain, Tim Reiterman, Nancy=\n",
      "=20\n",
      "Rivera-Brooks, Julie Tamaki, Nancy Vogel and Rone Tempest contributed to th=\n",
      "is=20\n",
      "story.=20\n",
      "--------------------------------------------------------------------------\n",
      "NEWS ANALYSIS\n",
      "Davis Keeps His Distance From Utility Rate Hikes=20\n",
      "\n",
      "By DAN MORAIN, Times Staff Writer=20\n",
      "\n",
      "?????SACRAMENTO--Gov. Gray Davis, convinced that he can't win votes by=20\n",
      "delivering bad news, avoided the task of telling Californians what virtuall=\n",
      "y=20\n",
      "all experts have been saying for months--that electricity bills are going u=\n",
      "p.\n",
      "?????Indeed, on Tuesday, the Democratic governor quickly moved to distance=\n",
      "=20\n",
      "himself from a rate hike of as much as 46% that was approved by the=20\n",
      "California Public Utilities Commission, led by his appointee Loretta Lynch.\n",
      "?????That and other recent moves have thrown Davis' credibility into=20\n",
      "question--even in his own Democratic Party.\n",
      "?????\"If you're the governor,\" said political consultant Darry Sragow, \"you=\n",
      "=20\n",
      "don't want to take the blame for a massive rate increase that is not going =\n",
      "to=20\n",
      "sit well with a lot of voters. You want to be in a position where you can=\n",
      "=20\n",
      "say, 'I didn't have a choice; my back was against the wall.' \"\n",
      "?????Davis describes his view of the situation as \"optimistic.\" He has call=\n",
      "ed=20\n",
      "it an \"energy challenge,\" not a crisis, even as the state's two largest=20\n",
      "utilities border on bankruptcy, and the state experienced blackouts a week=\n",
      "=20\n",
      "ago, in mid-March, when electricity use was 50% less than it will be in the=\n",
      "=20\n",
      "summer, when demand is highest.\n",
      "?????Davis and his aides say the governor is fully engaged in the problem,=\n",
      "=20\n",
      "understands its many dimensions and is working long hours to solve it. But=\n",
      "=20\n",
      "the crisis wears on, and an increasing number of lawmakers, including fello=\n",
      "w=20\n",
      "Democrats, are alarmed.\n",
      "?????\"It's an energy crisis, not an energy challenge,\" said Senate Energy=\n",
      "=20\n",
      "Committee Chairwoman Debra Bowen (D-Marina del Rey), who began warning of a=\n",
      "=20\n",
      "coming energy shortage 13 months ago.\n",
      "?????\"This is a major crisis,\" Sen. Don Perata (D-Alameda) said, \"and it=20\n",
      "would appear he is overwhelmed by it.\"\n",
      "?????\"If you don't face reality,\" said Sen. John Vasconcellos (D-Santa=20\n",
      "Clara), \"you can't possibly improve it. His posture has been baffling. I=20\n",
      "lament what is going on. This is my beloved state, and we're in serious=20\n",
      "peril, and it scares me.\"\n",
      "?????Many academics, utility executives, Wall Street analysts and legislato=\n",
      "rs=20\n",
      "concluded weeks and months ago that rate hikes were necessary, given the=20\n",
      "soaring wholesale price of electricity. Even Davis has acknowledged that a=\n",
      "=20\n",
      "rate hike might solve the situation.\n",
      "?????\"Believe me,\" Davis said Feb. 16, \"if I wanted to raise rates, I could=\n",
      "=20\n",
      "have solved this problem in 20 minutes.\"\n",
      "?????But in a reaction that added to lawmakers' bewilderment, Davis said in=\n",
      " a=20\n",
      "statement Tuesday that the PUC action was \"premature because we do not have=\n",
      "=20\n",
      "all the appropriate finance numbers necessary to make a decision.\"\n",
      "?????\"Until we do,\" the statement continued, \"I cannot support any rate=20\n",
      "increase.\"\n",
      "?????In taking steps to raise rates, Lynch may well end up taking substanti=\n",
      "al=20\n",
      "flak for her governor.\n",
      "?????\"Recent polls show the PUC is very unpopular in this state,\" Sragow=20\n",
      "said. \"Gray Davis is still popular. Voters clearly don't see a connection=\n",
      "=20\n",
      "between the governor and the PUC.\"\n",
      "?????Davis' aides say the governor didn't talk to Lynch before she acted.=\n",
      "=20\n",
      "Lynch, a frequent visitor to the Capitol, was in Sacramento late last week=\n",
      "=20\n",
      "and again Tuesday. She may not have spoken to Davis, but she did make other=\n",
      "=20\n",
      "top administration officials aware of her plans.\n",
      "?????Lynch could have decided to act on her own. The state Constitution giv=\n",
      "es=20\n",
      "the commission authority to raise utility rates. But the governor is well=\n",
      "=20\n",
      "known as a micro-manager, involved in the most minute details of state=20\n",
      "governance. Davis repeatedly has said he expects all his appointees to \"thi=\n",
      "nk=20\n",
      "like I think,\" and not to act on their own.\n",
      "?????As recently as last week, Davis directed that the PUC take specific=20\n",
      "actions to help ensure that alternative energy producers get paid.\n",
      "?????Davis has said the Legislature's job is to implement his vision. He ha=\n",
      "s=20\n",
      "said judges he appoints who later break from him on major issues should=20\n",
      "resign. His appointees \"should not be free agents.\"\n",
      "?????So he strained his credibility by saying he was unaware that Lynch wou=\n",
      "ld=20\n",
      "move to raise rates Tuesday, particularly given events that occurred late=\n",
      "=20\n",
      "last week.\n",
      "?????On Friday, Davis Finance Director Tim Gage and Susan Kennedy, one of h=\n",
      "is=20\n",
      "top aides, met with Assembly leaders behind closed doors. At the meeting, a=\n",
      "=20\n",
      "participant warned that rates might have to rise 50% to 100%.\n",
      "?????Also Friday, a top aide to Davis told The Times that Davis' staff was=\n",
      "=20\n",
      "convinced that a rate hike was necessary, even as Davis continued to public=\n",
      "ly=20\n",
      "oppose it. On Sunday, a top Davis aide referred reporters to Lynch, who=20\n",
      "confirmed that she would move to raise rates Tuesday.\n",
      "?????Two months ago, Harry Snyder of Consumers Union was defending Davis.=\n",
      "=20\n",
      "Now, Snyder, who has worked with Davis on various issues for decades, is,=\n",
      "=20\n",
      "like many others, raising questions about the governor's credibility, his=\n",
      "=20\n",
      "penchant for secrecy and his competence.\n",
      "?????\"Something is wrong,\" Snyder said. \"Either he is over his depth, or it=\n",
      "=20\n",
      "is hubris, or it is the great river in Egypt, 'Denial.' \"\n",
      "?????Here are some of the governor's statements and actions, since called=\n",
      "=20\n",
      "into question:\n",
      "?????* Davis held a news conference Feb. 23 to say he had reached an=20\n",
      "agreement on a rescue plan with Southern California Edison that included a=\n",
      "=20\n",
      "deal to buy its massive transmission system for $2.76 billion. A final=20\n",
      "package, he said, would be announced within a week, not only with Edison bu=\n",
      "t=20\n",
      "with San Diego Gas and Electric.\n",
      "?????A month later, a top utility executive described the talks as=20\n",
      "\"dreadfully slow.\" Negotiations over a similar deal with Pacific Gas &=20\n",
      "Electric, which he said back in February would be completed by now, are eve=\n",
      "n=20\n",
      "further behind.\n",
      "?????* Davis has refused to release details about contracts he signed with=\n",
      "=20\n",
      "independent generators to buy power. He said, however, that the contracts=\n",
      "=20\n",
      "accounted for 6,000 to 7,000 megawatts of electricity this summer, a=20\n",
      "significant amount that would help protect the state from having to buy muc=\n",
      "h=20\n",
      "of its power on the far more costly spot market.\n",
      "?????Last week, when he released some scant details in a report on the=20\n",
      "contracts, experts who read it concluded that California would be forced to=\n",
      "=20\n",
      "buy half or more of its electricity on the expensive spot market.\n",
      "?????* Davis brushed aside reporters' questions last week about the blackou=\n",
      "ts=20\n",
      "that rolled across the state. He remained cloistered as the outages continu=\n",
      "ed=20\n",
      "for a second day, emerging only after they ended to announce a plan to ensu=\n",
      "re=20\n",
      "that alternative energy producers would be paid. A week after he called on=\n",
      "=20\n",
      "lawmakers to quickly approve a bill that would help get them paid, the=20\n",
      "legislation remains stalled.\n",
      "?????\"It is inept at all levels and doesn't encourage any confidence that h=\n",
      "e=20\n",
      "can handle it,\" Snyder said. \"Moreover, it doesn't encourage us that he is=\n",
      "=20\n",
      "going to tell the truth.\"\n",
      "?????The energy crisis is perilous for Davis' reelection effort next year.=\n",
      "=20\n",
      "But the governor's strategy of being unremittingly optimistic and going to=\n",
      "=20\n",
      "great lengths to maintain his opposition to rate hikes, could be to his=20\n",
      "benefit when he faces voters.\n",
      "?????\"Ethically,\" said UC Berkeley political scientist Bruce Cain, \"not bei=\n",
      "ng=20\n",
      "forthcoming is worse. Politically, the more he can obfuscate, the more like=\n",
      "ly=20\n",
      "it is some people will buy the obfuscation. . . . The reason we see=20\n",
      "obfuscation is that obfuscation works.\"\n",
      "?????Not all politicians follow that maxim.\n",
      "?????Davis' predecessor, Republican Gov. Pete Wilson, held a news conferenc=\n",
      "e=20\n",
      "in 1991, as California plunged into a recession, to announce that he was=20\n",
      "pushing for what became the largest tax increase in state history, calling =\n",
      "it=20\n",
      "necessary to help erase a $14-billion deficit. Top advisors had urged him t=\n",
      "o=20\n",
      "leave the delivery of such bad news to others.\n",
      "?????\"Any time you deliver bad news to voters you're taking a risk,\" said D=\n",
      "an=20\n",
      "Schnur, who was Wilson's spokesman. \"But it is a much bigger risk to allow =\n",
      "a=20\n",
      "situation to fester, without preparing voters for decisions that have to be=\n",
      "=20\n",
      "made.\n",
      "?????\"Nobody likes pain,\" Schnur said. \"But they are much more willing to=\n",
      "=20\n",
      "tolerate it if they understand why it is necessary.\"\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "Federal Energy Agency Unlikely to Order Refunds=20\n",
      "\n",
      "Power: Regulatory panel says much of the overcharge alleged by Cal-ISO is=\n",
      "=20\n",
      "beyond its jurisdiction, and questions whether the rest is \"excessive.\"=20\n",
      "\n",
      "By MEGAN GARVEY and RICHARD SIMON, Times Staff Writers=20\n",
      "\n",
      "?????WASHINGTON--What are the chances that federal regulators will order=20\n",
      "electricity wholesalers to refund $6.3 billion in alleged overcharges?\n",
      "?????Zero.\n",
      "?????Officials of the Federal Energy Regulatory Commission have=20\n",
      "already made clear that nearly $3 billion--almost half the total that=20\n",
      "California's grid operator said last week was excessive--is off the table=\n",
      "=20\n",
      "because a statute of limitations puts alleged overcharges that took place=\n",
      "=20\n",
      "before Oct. 2 beyond their reach.=20\n",
      "?????As for the rest, lawyers familiar with the agency and industry observe=\n",
      "rs=20\n",
      "note that FERC has consistently hesitated to use its full legal authority t=\n",
      "o=20\n",
      "enforce \"fair and reasonable\" rates because of difficulty in defining that=\n",
      "=20\n",
      "phrase. Some within FERC question how much of the \"excessive\" rates=20\n",
      "documented by the California Independent System Operator fall beyond FERC's=\n",
      "=20\n",
      "jurisdiction--for example, rates charged by municipal power suppliers that=\n",
      "=20\n",
      "are not regulated by the federal agency.\n",
      "?????So far, FERC has ordered electricity wholesalers to justify or refund=\n",
      "=20\n",
      "only $124 million in alleged overcharges--just 5% of the amount that the=20\n",
      "Cal-ISO has identified. That sum deals only with rates charged in January a=\n",
      "nd=20\n",
      "February 2001 during Stage 3 alerts--periods when electricity reserves fall=\n",
      "=20\n",
      "below 1.5%.\n",
      "?????FERC's approach to the alleged California overcharges will probably no=\n",
      "t=20\n",
      "become any more aggressive as a result of the two new members it will soon=\n",
      "=20\n",
      "have. President Bush has picked two state utility regulators to fill=20\n",
      "vacancies on the five-member commission, the White House announced Tuesday.\n",
      "?????The president plans to nominate Pat Wood III, chairman of the Texas=20\n",
      "Utility Commission and a Bush ally, and Nora Mead Brownell, a member of the=\n",
      "=20\n",
      "Pennsylvania Public Utility Commission, to the FERC governing board.\n",
      "?????Administration officials have indicated that Wood, 38, would assume th=\n",
      "e=20\n",
      "FERC chairmanship, a position now held by Curt Hebert Jr., a staunch advoca=\n",
      "te=20\n",
      "of energy deregulation.\n",
      "\n",
      "?????Shake-Up at FERC Was Expected\n",
      "?????The shake-up has been expected because of continuing complaints that=\n",
      "=20\n",
      "FERC has not been aggressive enough in dealing with high energy prices and=\n",
      "=20\n",
      "electricity shortages in California and elsewhere in the West. Hebert in=20\n",
      "particular has angered California officials with his opposition to price=20\n",
      "controls on wholesale electricity sales.\n",
      "?????Although the administration wants to appear sympathetic to California'=\n",
      "s=20\n",
      "plight, replacing Hebert has been a problem because he is a protege of Sena=\n",
      "te=20\n",
      "Majority Leader Trent Lott (R-Miss.).\n",
      "?????Cal-ISO officials--who pointed out that their $6.3-billion figure was=\n",
      "=20\n",
      "part of a study calling for stronger market controls, not for refunds--may=\n",
      "=20\n",
      "still formally ask FERC to augment the refund order for January and Februar=\n",
      "y.=20\n",
      "FERC ordered wholesalers to justify $124 million worth of charges shortly=\n",
      "=20\n",
      "after Cal-ISO asked for $550 million in refunds for December and January.\n",
      "?????FERC's refund process reflects the ambiguities facing federal regulato=\n",
      "rs=20\n",
      "who oversee an electricity market that was designed to follow the law of=20\n",
      "supply and demand. At stake may be the final bill for electricity used to=\n",
      "=20\n",
      "date in California as well as future rates.\n",
      "?????And the debate over the role refunds should play in addressing=20\n",
      "skyrocketing wholesale electricity prices underscores diverging philosophie=\n",
      "s=20\n",
      "of what went wrong in the California market and how to fix it. What role=20\n",
      "should the government--specifically FERC, an agency with a mandate to ensur=\n",
      "e=20\n",
      "\"fair and reasonable\" rates--play?\n",
      "?????\"The whole notion that there's such a thing as a just price is kind of=\n",
      "=20\n",
      "an anachronism from New Deal days,\" said Pietro Nivola, a Brookings Institu=\n",
      "te=20\n",
      "scholar who studies regulatory politics. \"Especially as we move closer to=\n",
      "=20\n",
      "some form of deregulation, we've come to understand that the price is what=\n",
      "=20\n",
      "the traffic will bear. In this case, utilities simply have to be able to ge=\n",
      "t=20\n",
      "higher rates at the retail end.\"\n",
      "?????But others say FERC has failed to assert its authority, allowing the=\n",
      "=20\n",
      "wholesalers to operate with impunity.\n",
      "?????\"The message FERC has sent so far is, as long as you leave 10 cents of=\n",
      "=20\n",
      "every dollar in the bank we won't pull the alarm,\" said Mark Cooper, direct=\n",
      "or=20\n",
      "of research for the Consumer Federation of America.\n",
      "?????Some on Capitol Hill say a massive refund order by FERC--while highly=\n",
      "=20\n",
      "unlikely--would go a long way toward reining in wholesale prices. But FERC=\n",
      "=20\n",
      "officials and many economists say that view is myopic and may do more harm=\n",
      "=20\n",
      "than good, potentially discouraging new suppliers from entering a market=20\n",
      "already too tight. The memory of the 1970s oil crisis and a prevalent view=\n",
      "=20\n",
      "among economists that government intervention made matters worse has inform=\n",
      "ed=20\n",
      "the views of some longtime government officials and observers.\n",
      "\n",
      "?????Refunds Called 'Inferior Remedy'\n",
      "?????A clear sign of FERC's reluctance to call for refunds came late last=\n",
      "=20\n",
      "year, when the commission issued its initial proposition for remedies for t=\n",
      "he=20\n",
      "California price increases. FERC officials said at the time, \"Refunds may b=\n",
      "e=20\n",
      "an inferior remedy from a market perspective and not the fundamental soluti=\n",
      "on=20\n",
      "to any problems occurring in California markets.\"\n",
      "?????The decision by FERC to scrutinize electricity charges only during Sta=\n",
      "ge=20\n",
      "3 alerts in January and February was harshly criticized earlier this month =\n",
      "by=20\n",
      "some lawmakers and utility company officials. They complained that the narr=\n",
      "ow=20\n",
      "focus eliminated tens of millions of dollars in charges some observers=20\n",
      "believed were outrageous.\n",
      "?????But even the $124 million questioned by FERC has been defended by the=\n",
      "=20\n",
      "wholesale companies. In briefs filed with the agency last week, they said=\n",
      "=20\n",
      "rates exceeding $430 per megawatt-hour--compared with a pre-crisis average =\n",
      "of=20\n",
      "$30 per megawatt-hour--were justified by the increased credit risk=20\n",
      "wholesalers took in selling to near-bankrupt utilities that have billions o=\n",
      "f=20\n",
      "dollars in outstanding debts.\n",
      "?????Duke Energy, which FERC has said overcharged California by $20 million=\n",
      ",=20\n",
      "signaled some room for compromise in its compliance filing.=20\n",
      "?????\"We would be willing to forgo collection of credit premiums for these=\n",
      "=20\n",
      "months provided we were paid the FERC clearing price,\" company president Ji=\n",
      "m=20\n",
      "Donnell said this week. \"If we are not assured payment, the credit premiums=\n",
      "=20\n",
      "are obviously appropriate, and we would reserve our right to collect the=20\n",
      "entire amount of the bids that are subject to the FERC order.\"\n",
      "?????Both of Bush's Republican nominees to the commission are expected to w=\n",
      "in=20\n",
      "Senate confirmation. But they can expect a grilling from Sen. Dianne=20\n",
      "Feinstein (D-Calif.), who has accused power suppliers of price-gouging and=\n",
      "=20\n",
      "FERC of doing little to stop it.=20\n",
      "?????\"Certainly, my vote would be dependent on whether they're going to be=\n",
      "=20\n",
      "responsive to the present situation,\" Feinstein said Tuesday, grousing that=\n",
      "=20\n",
      "the administration has been \"recalcitrant to the point of arrogance.\"\n",
      "?????Feinstein met Tuesday with Vice President Dick Cheney, head of a White=\n",
      "=20\n",
      "House energy task force, but she described the meeting as \"disappointing.\"=\n",
      "=20\n",
      "Feinstein also sent a letter to Bush asking to meet with him on the=20\n",
      "California crisis, and adding a hand-written note: \"The West needs your hel=\n",
      "p.\"\n",
      "?????A spokesman for Gov. Gray Davis said he hoped the new commissioners=20\n",
      "would have a \"more sympathetic ear\" for the concerns of the Western states.\n",
      "?????But word of the nominations came as Bush reiterated his opposition to=\n",
      "=20\n",
      "price controls. \"Price controls contributed to the gas lines of the 1970s,=\n",
      "=20\n",
      "and the United States will not repeat the mistake again,\" he said in a spee=\n",
      "ch=20\n",
      "in Kalamazoo, Mich.\n",
      "?????The nominees drew praise from the Electric Power Supply Assn. The grou=\n",
      "p=20\n",
      "expressed confidence that they would be \"strong proponents of completing th=\n",
      "e=20\n",
      "much-needed transition to truly competitive and vibrant power markets all=\n",
      "=20\n",
      "across the country.\"\n",
      "?????Neither Wood nor Brownell could be reached for comment. Wood said in a=\n",
      "=20\n",
      "statement: \"On our best day as regulators, we cannot deliver benefits to=20\n",
      "customers as well as a functional market can. But the market must work righ=\n",
      "t=20\n",
      "first.\"\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------------------------------------------\n",
      "\n",
      "Wednesday, March 28, 2001=20\n",
      "A Painful Step=20\n",
      "Electricity rate increase may help, but Gov. Davis is yet to offer an=20\n",
      "understandable plan to end the crisis.=20\n",
      "\n",
      "?????By approving a whopping rate hike for big power users, the state Publi=\n",
      "c=20\n",
      "Utilities Commission finally has recognized the grim reality of the=20\n",
      "electricity market in California, even if Gov. Gray Davis has not.=20\n",
      "Electricity costs more these days, even aside from the piratical rates=20\n",
      "charged by big generating companies at times during this emergency.=20\n",
      "?????The increase, while probably necessary, will not by itself solve the=\n",
      "=20\n",
      "crisis or prevent rolling blackouts this summer. The two biggest private=20\n",
      "utilities, Southern California Edison and Pacific Gas & Electric, still are=\n",
      "=20\n",
      "saddled with as much as $13 billion in debt for past power purchases. State=\n",
      "=20\n",
      "government still is shelling out some $50 million in daily power purchases=\n",
      "=20\n",
      "and has committed as much as $40 billion to future power contracts. Davis=\n",
      "=20\n",
      "continues to negotiate a takeover of the utilities' share of the state powe=\n",
      "r=20\n",
      "transmission grid in exchange for paying off some of the utilities' debts.=\n",
      "=20\n",
      "?????Despite all the hand-wringing and arm-waving, nothing that state=20\n",
      "government has done since January has provided any significant new power=20\n",
      "generation or conservation. Consumers are right to wonder what happened to =\n",
      "a=20\n",
      "promised statewide conservation campaign. After first ducking the power=20\n",
      "problem, Davis launched a flurry of initiatives early in the year, most of=\n",
      "=20\n",
      "them still works in progress. But he still has not given the people an=20\n",
      "understandable, coherent plan for getting out of this fix. Worse, he refuse=\n",
      "s=20\n",
      "even to say how much the state is spending on power even as he plunges=20\n",
      "Sacramento into the power business for decades into the future.=20\n",
      "?????The irony of Tuesday's rate increase is that the state might have dodg=\n",
      "ed=20\n",
      "much of the mess if it had raised rates three months ago. But Davis, seekin=\n",
      "g=20\n",
      "reelection next year, insisted that the crisis be solved \"within the existi=\n",
      "ng=20\n",
      "rate structure.\" Late last week he attended a political fund-raiser at a Pa=\n",
      "lm=20\n",
      "Springs golf club while his staff was being briefed on the need for higher=\n",
      "=20\n",
      "rates, literally distancing himself from the act.=20\n",
      "?????The rate increase, which will be drafted over the next 30 days, affect=\n",
      "s=20\n",
      "customers of Edison and PG&E but not residents of Los Angeles or other citi=\n",
      "es=20\n",
      "with full municipal power systems. The stated intent is that bigger users,=\n",
      "=20\n",
      "particularly businesses, will pay higher rates and be encouraged to save=20\n",
      "major amounts of power during the summer. Under one scenario, nearly half o=\n",
      "f=20\n",
      "households--low and medium users--would pay no more than they do now. The=\n",
      "=20\n",
      "boost is also intended to compensate taxpayers for the state's daily power=\n",
      "=20\n",
      "purchases, although there is no assurance yet that the new rates will meet=\n",
      "=20\n",
      "the full cost. The increase should be structured to give ratepayers some=20\n",
      "relief if lawsuits against power generators' exorbitant pricing are=20\n",
      "successful in winning refunds.=20\n",
      "?????Those skeptical ratepayers who believe that the crisis is just a big=\n",
      "=20\n",
      "rip-off by the utilities and the energy companies will consider the increas=\n",
      "e=20\n",
      "more of the same. You can't blame them, considering the lack of straight ta=\n",
      "lk=20\n",
      "from the governor, including his dodging of the rate issue.=20\n",
      "?????However, most state experts, including Davis' own utilities=20\n",
      "commissioners, say the rate increase is a critical first step in solving=20\n",
      "California's power problems. If Davis has a plan to fix the problem some=20\n",
      "other way, he should present it in detail to the people of California now.=\n",
      "=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------\n",
      "PUC Votes To Jack Up Power Rates=20\n",
      "Tiered increases approved as protests disrupt meeting=20\n",
      "David Lazarus, Chronicle Staff Writer\n",
      "Wednesday, March 28, 2001=20\n",
      ",2001 San Francisco Chronicle=20\n",
      "URL:=20\n",
      "http://www.sfgate.com/cgi-bin/article.cgi?file=3D/chronicle/archive/2001/03=\n",
      "/28/M\n",
      "N231034.DTL=20\n",
      "The bill finally came due for California's experiment with electricity=20\n",
      "deregulation as consumers were hit yesterday with the largest rate increase=\n",
      "=20\n",
      "in state history, an average 40 percent jump in monthly power costs.=20\n",
      "The immediate rate hike was unanimously approved by the Public Utilities=20\n",
      "Commission during a rowdy meeting that was repeatedly interrupted by chanti=\n",
      "ng=20\n",
      "and jeers from protesters.=20\n",
      "Highway Patrol officers were directed by PUC President Loretta Lynch to=20\n",
      "remove several demonstrators from the packed San Francisco auditorium.=20\n",
      "\"These are extraordinary moments in California's history, and extraordinary=\n",
      "=20\n",
      "moments demand extraordinary courage,\" PUC member Geoffrey Brown said befor=\n",
      "e=20\n",
      "the rate increase was passed.=20\n",
      "The commissioners approved a new 30 percent increase and made permanent a=\n",
      "=20\n",
      "\"temporary\" 10 percent increase adopted in January. This will allow the=20\n",
      "state's two largest utilities to charge customers an extra $4.8 billion a=\n",
      "=20\n",
      "year.=20\n",
      "PUC members said higher rates will help overcome the wide disparity between=\n",
      "=20\n",
      "skyrocketing costs charged by power companies for electricity and the=20\n",
      "relatively low amount paid by consumers under California's 1996 deregulatio=\n",
      "n=20\n",
      "law.=20\n",
      "However, consumers' bills will not go up until a \"tiered\" rate system can b=\n",
      "e=20\n",
      "introduced, requiring those who use more power to pay higher fees. That=20\n",
      "system is expected to be in place by May.=20\n",
      "Gov. Gray Davis, who has tried to distance himself from the PUC's move sinc=\n",
      "e=20\n",
      "it was unveiled Monday, said in a statement yesterday that the rate increas=\n",
      "e=20\n",
      "was \"premature\" and that he did not support the decision.=20\n",
      "\"While I have opposed rate increases, if it becomes clear that a rate=20\n",
      "increase is absolutely necessary for the good of the state, I will support=\n",
      "=20\n",
      "one that is fair and do my duty to convince Californians of its necessity,\"=\n",
      "=20\n",
      "he said.=20\n",
      "Several PUC members acknowledged that such a huge rate increase might not=\n",
      "=20\n",
      "have been necessary had the commission acted sooner to remedy California's=\n",
      "=20\n",
      "power woes.=20\n",
      "\"We should have done this in January,\" Brown said, \"but we didn't have enou=\n",
      "gh=20\n",
      "information.\"=20\n",
      "\"This rate increase is long overdue,\" said Richard Bilas, who also sits on=\n",
      "=20\n",
      "the commission. \"We are finally transfusing some blood.\"=20\n",
      "CONSUMERS PROTEST\n",
      "If so, the donor -- ratepayers -- did not go willingly to the operating=20\n",
      "theater. A number of citizens rose to speak at yesterday's meeting, and not=\n",
      "=20\n",
      "one was in favor of higher power rates.=20\n",
      "\"This is being shoved down the people's throats,\" said Dorothy Diez, an=20\n",
      "elderly San Francisco resident. \"Why don't they put the bills where they=20\n",
      "belong and not on my back?\"=20\n",
      "She and others called on state officials to take a harder line with out-of-=\n",
      "=20\n",
      "state power generators, which have reaped windfall profits by charging sky-=\n",
      "=20\n",
      "high wholesale prices for their electricity.=20\n",
      "Consumer groups urged the PUC to support special taxes on the generators'=\n",
      "=20\n",
      "earnings or even seizure of the companies' California power plants.=20\n",
      "\"Your actions are reprehensible,\" Ross Mirkarimi, a public-power supporter,=\n",
      "=20\n",
      "told the commission. \"Do not do Gov. Davis' dirty work.\"=20\n",
      "The governor did not attend yesterday's meeting, but his presence nonethele=\n",
      "ss=20\n",
      "loomed large over the proceedings.=20\n",
      "Consumer advocates derided Davis' claims this week that he played no role i=\n",
      "n=20\n",
      "the decision to raise rates. They noted that the governor's appointees hold=\n",
      " a=20\n",
      "majority on the PUC and worked closely with Davis on a variety of past=20\n",
      "issues.=20\n",
      "Davis' staff was briefed on the rate hike before the PUC publicized its=20\n",
      "decision.=20\n",
      "'DAVIS CONTROLS PUC'=20\n",
      "\"Gov. Davis controls the PUC,\" said Doug Heller, assistant organizing=20\n",
      "director of the Foundation for Taxpayer and Consumer Rights. \"He is=20\n",
      "responsible for the rate hike. We are paying more because of the failure of=\n",
      "=20\n",
      "his leadership.\"=20\n",
      "\"It seems absurd that Gov. Davis is saying his own agency is acting without=\n",
      "=20\n",
      "his consent,\" said Susannah Churchill, head of energy policy for the=20\n",
      "California Public Interest Research Group. \"He's just looking for political=\n",
      "=20\n",
      "cover.\"=20\n",
      "The PUC decision was made on a day when California's grid operators declare=\n",
      "d=20\n",
      "a Stage 2 alert, the first in a week, indicating that power reserves had=20\n",
      "dropped below 5 percent.=20\n",
      "When tiered rates are implemented, customers of Pacific Gas and Electric Co=\n",
      ".=20\n",
      "will see rates go up between 9 percent for \"medium\" power users and 36=20\n",
      "percent for \"heavy\" users.=20\n",
      "Those who can keep power consumption to within 130 percent of pre-=20\n",
      "established limits will see no increase beyond the 10 percent rate hike=20\n",
      "adopted in January.=20\n",
      "Customers of Southern California Edison will see rates increase between 8 a=\n",
      "nd=20\n",
      "27 percent.=20\n",
      "Bruce Foster, an Edison vice president, called the rate increase \"a step in=\n",
      "=20\n",
      "the right direction,\" but said further increases may be needed to defray=20\n",
      "soaring power costs this summer.=20\n",
      "PG&E spokesman Ron Low said the San Francisco utility needs to study the=20\n",
      "PUC's decision more closely before determining whether a 40 percent rate hi=\n",
      "ke=20\n",
      "will be sufficient.=20\n",
      "\"The PUC has done all it can,\" said Lynch, the PUC president. \"We have foug=\n",
      "ht=20\n",
      "back hard in every venue possible against unjust energy prices.\"=20\n",
      "NO FEDERAL HELP\n",
      "In the end, though, she said state regulators were forced to admit that no=\n",
      "=20\n",
      "help would be forthcoming from federal authorities in capping wholesale pow=\n",
      "er=20\n",
      "rates, and that generators would continue exploiting California's chronic=\n",
      "=20\n",
      "electricity shortage.=20\n",
      "\"We maintain a responsibility to keep the lights on,\" Lynch said.=20\n",
      "Along the same lines, the PUC approved a requirement that PG&E and Edison=\n",
      "=20\n",
      "repay the state Department of Water Resources for about $4 billion in power=\n",
      "=20\n",
      "purchases made on the cash-strapped utilities' behalf.=20\n",
      "The commission also passed a motion forcing PG&E and Edison to pay smaller=\n",
      "=20\n",
      "power generators for their output. Half of all such generators were shut do=\n",
      "wn=20\n",
      "during last week's blackouts because they had not been paid by the utilitie=\n",
      "s.=20\n",
      "However, the PUC is only requiring that utilities pay for future electricit=\n",
      "y=20\n",
      "purchases, and did not address the millions of dollars in payments still ow=\n",
      "ed=20\n",
      "the power companies.=20\n",
      "Ann MacLeod, executive director of the California Cogeneration Council, an=\n",
      "=20\n",
      "association of smaller generators, said that by avoiding the question of ba=\n",
      "ck=20\n",
      "debt, the PUC's move will do nothing to restore plants to operation.=20\n",
      "\"These generators will not come back online as a result of this decision,\"=\n",
      "=20\n",
      "she said. \"In fact, a greater number likely will have to drop offline.\"=20\n",
      "Protesters chanting, \"Hell no, we won't pay,\" urged consumers after the PUC=\n",
      "=20\n",
      "meeting to not include the higher fees in upcoming bills.=20\n",
      "\"This is just the beginning,\" said Medea Benjamin, who was among the=20\n",
      "demonstrators evicted from the earlier proceedings. \"Rates will go up and u=\n",
      "p=20\n",
      "and up.\"=20\n",
      "What's Next=20\n",
      "The McCain-Feingold campaign finance reform bill faces several more tests,=\n",
      "=20\n",
      "among them:=20\n",
      "- An amendment by Sen. Fred Thompson, R-Tenn., that would double the=20\n",
      "contributions that donors may make to candidates, PACs and some other=20\n",
      "organizations. A vote is expected today.=20\n",
      "- Sen. Mitch McConnell, the measure's chief foe, refused yesterday to agree=\n",
      "=20\n",
      "to allow a final vote on the legislation tomorrow. McConnell, R-Ky., left=\n",
      "=20\n",
      "open the possibility of using parliamentary maneuvers to block a final vote=\n",
      ".=20\n",
      "Chronicle News Services=20\n",
      "\n",
      "THE ENERGY CRUNCH\n",
      "Low to Medium Users=20\n",
      "(up to 130% of baseline)=20\n",
      "0%=20\n",
      "Medium Users=20\n",
      "(130% to 200% of baseline)=20\n",
      "9%=20\n",
      "Heavy Users=20\n",
      "(more than 200% of baseline)=20\n",
      "36%=20\n",
      "\n",
      "UNDERSTANDING YOUR BILL\n",
      "1. Total Electric Charges: The electric usage multiplied by the rate in=20\n",
      "dollars and cents.=20\n",
      "2. Legislated 10% Reduction: 10 percent reduction, financed with bond sale,=\n",
      "=20\n",
      "mandated by state law that set deregulation in motion. Remains in effect=20\n",
      "until no later than March 31, 2002.=20\n",
      "3. Net charges: The cost of electricity service after 10 percent reduction.=\n",
      "=20\n",
      "Breakdown of electric charges=20\n",
      "4. Electric Energy Charge: Average cost of buying electricity during billin=\n",
      "g=20\n",
      "period.=20\n",
      "5. Transmission: Cost of moving electricity from plants to distribution=20\n",
      "centers.=20\n",
      "6. Distribution: Cost of moving electricity from distribution centers to=20\n",
      "homes.=20\n",
      "7. Public Purpose Programs: Funds benefit programs such as affordable servi=\n",
      "ce=20\n",
      "for low-income families and conservation programs.=20\n",
      "8. Nuclear Decommissioning: Fee to restore plant sites to original conditio=\n",
      "n=20\n",
      "once shut down.=20\n",
      "9. Competition Transition Charge: Originally built into rates as a charge=\n",
      "=20\n",
      "above the cost of providing power. Called \"headroom,\" it was essentially an=\n",
      "=20\n",
      "extra fee to pay back utilities for their investments in power plants. At t=\n",
      "he=20\n",
      "moment, it's a negative number because the cost of buying power on the=20\n",
      "wholesale market is so much higher than the rate customers can be charged b=\n",
      "y=20\n",
      "law.=20\n",
      "10. Trust Transfer Amount: Fee to pay off bonds that financed 10 percent ra=\n",
      "te=20\n",
      "reduction.=20\n",
      "Baseline Information Box=20\n",
      "11. Baseline Quantities: The maximum amount of usage that can be billed at=\n",
      "=20\n",
      "the regular rate; varies by season, climate zone and heat source.=20\n",
      "12. Baseline Usage: Amount of electricity usage up to baseline quantity=20\n",
      "level, billed at regular rate.=20\n",
      "13. Over Baseline Usage: Amount of electricity used above baseline=20\n",
      "quantities, at a higher rate.=20\n",
      "HOW THINGS WOULD CHANGE\n",
      "The primary areas in which a customer bill would change under the new syste=\n",
      "m=20\n",
      "would be:=20\n",
      "1. The creation of a new line item in the Baseline Information box for=20\n",
      "electricity used 130 percent above the baseline. The line would reflect how=\n",
      "=20\n",
      "much was used at that level and the rate for that usage.=20\n",
      "2. A higher dollar amount in the Electric Energy Charge line item reflectin=\n",
      "g=20\n",
      "the increase cost from the rate hike. .=20\n",
      "SOURCE: PG&E Web site; interviews.=20\n",
      "E-mail David Lazarus at dlazarus@sfchronicle.com.=20\n",
      ",2001 San Francisco Chronicle ? Page?A - 1=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------\n",
      "Developments in California's power crisis=20\n",
      "The Associated Press\n",
      "Wednesday, March 28, 2001=20\n",
      ",2001 Associated Press=20\n",
      "URL:=20\n",
      "http://www.sfgate.com/cgi-bin/article.cgi?file=3D/news/archive/2001/03/28/s=\n",
      "tate0\n",
      "859EST0138.DTL=20\n",
      ", , -- (03-28) 05:59 PST Here is a look at developments in California's=20\n",
      "electricity crisis:<=20\n",
      "WEDNESDAY:< ?-- The state remains free of power alerts in the early morning=\n",
      " as power ?reserves stay above 7 percent. ?-- The Assembly resumes hearings=\n",
      " on California's high natural gas prices. ?-- The Assembly is expected to t=\n",
      "ry again to pass a bill that would let the ?state Public Utilities Commissi=\n",
      "on order Southern California Edison Co. and ?Pacific Gas & Electric Co. to =\n",
      "pay alternative-energy plants. ?TUESDAY:<=20\n",
      "-- The PUC votes unanimously to approve electricity rate hikes of up to 46=\n",
      "=20\n",
      "percent for customers of California's two largest utilities. The board vote=\n",
      "s=20\n",
      "5-0 to increase rates immediately for PG&E and Edison despite yells from=20\n",
      "protesters. The increases are the largest in California history.=20\n",
      "-- Stock in both utilities suffer slight losses after expectations of the=\n",
      "=20\n",
      "rate hikes Monday caused shares in PG&E Corp. to surge 29 percent and Ediso=\n",
      "n=20\n",
      "International's stock to gain 30 percent. Edison International closed down =\n",
      "39=20\n",
      "cents at $14.16 a share. PG&E Holding Corp., parent of Pacific Gas &=20\n",
      "Electric, closed down 55 cents at $13.20.=20\n",
      "-- In a Tuesday afternoon conference call, Southern California Edison says =\n",
      "it=20\n",
      "will begin paying interest to its bondholders after weeks of failure to pay=\n",
      ".=20\n",
      "Company authorities say they have not heard whether the news will boost the=\n",
      "ir=20\n",
      "credit rating and do not specify a dollar amount for monthly interest=20\n",
      "payments.=20\n",
      "-- The Los Angeles Department of Water and Power, which is not attached to=\n",
      "=20\n",
      "the state power grid and is not affected by deregulation, issues a statemen=\n",
      "t=20\n",
      "Tuesday assuring that customer rates ``will continue to remain stable.''=20\n",
      "-- Standard & Poors retains California's cautionary credit rating despite=\n",
      "=20\n",
      "state regulators' decision to raise electricity rates. The state has been o=\n",
      "n=20\n",
      "a credit watch ``with negative implications'' since it began spending $45=\n",
      "=20\n",
      "million a day to buy electricity for customers of two nearly bankrupt=20\n",
      "utilities. The crediw watch will remain in effect despite a rate increase=\n",
      "=20\n",
      "state regulators approved Tuesday.=20\n",
      "-- President Bush says the electricity price caps sought by California's Gr=\n",
      "ay=20\n",
      "Davis and other Western governors would worsen the region's energy crisis=\n",
      "=20\n",
      "instead of helping cure it. In a speech to the Kalamazoo Chamber of Commerc=\n",
      "e,=20\n",
      "Bush says price controls helped cause the gasoline crisis of the 1970s, and=\n",
      "=20\n",
      "he won't make the same mistake.=20\n",
      "-- Joseph Fichera, the governor's chief negotiator on the purchase of=20\n",
      "transmission lines, says he is exchanging drafts of proposed agreements wit=\n",
      "h=20\n",
      "Edison officials but won't say how soon a final agreement may be reached. H=\n",
      "e=20\n",
      "says negotiations are continuing with PG&E and San Diego Gas & Electric.=20\n",
      "-- State agencies announce a March 29 workshop in Ontario for developers wh=\n",
      "o=20\n",
      "hope to build peaking power plants in time for this summer. The event is=20\n",
      "similar to one held two weeks ago in Sacramento. Peaking plants are used fo=\n",
      "r=20\n",
      "short periods when power supplies run low.=20\n",
      "-- California's Independent System Operator declares a Stage 2 alert when=\n",
      "=20\n",
      "electricity reserves were approaching only 5 percent of available power. IS=\n",
      "O=20\n",
      "spokeswoman Stephanie McCorkle says the alert was triggered when 1,000=20\n",
      "megawatts was unexpectedly lost from the Pacific Northwest. WHAT'S NEXT:< ?=\n",
      "-- The Davis administration continues negotiations with Edison, PG&E and Sa=\n",
      "n ?Diego Gas & Electric Co. over state acquisition of their transmission li=\n",
      "nes.< ?THE PROBLEM:<=20\n",
      "High demand, high wholesale energy costs, transmission glitches and a tight=\n",
      "=20\n",
      "supply worsened by scarce hydroelectric power in the Northwest and=20\n",
      "maintenance at aging California power plants are all factors in California'=\n",
      "s=20\n",
      "electricity crisis.=20\n",
      "Edison and PG&E say they've lost nearly $14 billion since June to high=20\n",
      "wholesale prices that the state's electricity deregulation law bars them fr=\n",
      "om=20\n",
      "passing onto ratepayers, and are close to bankruptcy.=20\n",
      "Electricity and natural gas suppliers, scared off by the two companies' poo=\n",
      "r=20\n",
      "credit ratings, are refusing to sell to them, leading the state in January =\n",
      "to=20\n",
      "start buying power for the utilities' nearly 9 million residential and=20\n",
      "business customers.=20\n",
      ",2001 Associated Press ?=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------\n",
      "Energy Nominees Would Give GOP Majority on Panel=20\n",
      "Carolyn Lochhead, Chronicle Washington Bureau\n",
      "Wednesday, March 28, 2001=20\n",
      ",2001 San Francisco Chronicle=20\n",
      "URL:=20\n",
      "http://www.sfgate.com/cgi-bin/article.cgi?file=3D/chronicle/archive/2001/03=\n",
      "/28/M\n",
      "N139149.DTL=20\n",
      "Washington -- President Bush nominated two new commissioners to the nation'=\n",
      "s=20\n",
      "top energy regulatory body yesterday, retaining the current chairman who ha=\n",
      "s=20\n",
      "criticized California's handling of its energy crisis.=20\n",
      "Bush tapped a fellow Texan, Patrick Henry Wood III, chairman of the Texas=\n",
      "=20\n",
      "Public Utilities Commission, and Nora Mead Brownell, a Pennsylvania public=\n",
      "=20\n",
      "utility commissioner, to fill two vacancies on the five-member Federal Ener=\n",
      "gy=20\n",
      "Regulatory Commission.=20\n",
      "The appointments would give Republicans a 3-to-2 majority on FERC, as the=\n",
      "=20\n",
      "commission is known, and probably solidify the agency's opposition to=20\n",
      "imposing wholesale energy price caps sought by California Gov. Gray Davis a=\n",
      "nd=20\n",
      "Sen. Dianne Feinstein, both Democrats.=20\n",
      "Both nominations require Senate confirmation, and Feinstein indicated=20\n",
      "yesterday that she might oppose the nominations.=20\n",
      "\"I would hope they would come in and talk with me,\" she said. \"Certainly my=\n",
      "=20\n",
      "vote would be dependent on whether they would be responsive to the present=\n",
      "=20\n",
      "situation. Up to this point, the administration has been recalcitrant to th=\n",
      "e=20\n",
      "point of arrogance. If that is the case with these new appointees as well,=\n",
      "=20\n",
      "they won't have my vote.\"=20\n",
      "FERC regulates wholesale power markets and interstate natural gas pipelines=\n",
      ".=20\n",
      "Curt Hebert, named chairman by Bush earlier this year, will remain in his=\n",
      "=20\n",
      "post. Although Hebert's outspoken free-market views are in sync with the=20\n",
      "administration's, there had been speculation that Bush would tap Wood for t=\n",
      "he=20\n",
      "top job. Some thought Bush might want a familiar ally to head an agency tha=\n",
      "t=20\n",
      "used to be a regulatory backwater but now is among the most politically=20\n",
      "sensitive spots in the administration.=20\n",
      "Wood has overseen electric utility deregulation in Texas, which is schedule=\n",
      "d=20\n",
      "to open its market to competition Jan. 1, 2002, but based on a model that=\n",
      "=20\n",
      "differs significantly from California's scheme. Texas has attracted ample n=\n",
      "ew=20\n",
      "power generation since 1995, and is experimenting with partial retail=20\n",
      "competition for commercial users.=20\n",
      "Brownell helped oversee Pennsylvania's electricity deregulation, which is=\n",
      "=20\n",
      "cited as a successful contrast to California's experiment.=20\n",
      "FERC has come under heavy political fire from California Democrats for not=\n",
      "=20\n",
      "cracking down hard enough on alleged price gouging of California utilities =\n",
      "by=20\n",
      "out-of-state energy producers.=20\n",
      "Hebert has been sharply critical of California's response to its electricit=\n",
      "y=20\n",
      "mess, once describing Davis' plan to take over the utilities' transmission=\n",
      "=20\n",
      "lines as \"nationalization.\" He is adamantly opposed to price controls and=\n",
      "=20\n",
      "enjoys the strong backing of fellow Mississippian and Senate Majority Leade=\n",
      "r=20\n",
      "Trent Lott, who pressured the White House to keep him.=20\n",
      "Hebert and Energy Secretary Spencer Abraham have voiced strong opposition t=\n",
      "o=20\n",
      "price controls on wholesale electricity, arguing that would worsen=20\n",
      "California's blackouts by discouraging electricity sales from outside the=\n",
      "=20\n",
      "state.=20\n",
      "Vice President Dick Cheney, who met informally with about 10 senators in hi=\n",
      "s=20\n",
      "Senate office yesterday, also restated the administration's opposition to=\n",
      "=20\n",
      "price caps, telling Feinstein price controls would not work and citing thei=\n",
      "r=20\n",
      "failure in the Nixon administration.=20\n",
      "Feinstein, who has said she equates helping California with the willingness=\n",
      "=20\n",
      "to impose federal price caps, called Cheney's response \"very disappointing.=\n",
      "\"=20\n",
      "\"There was really no overture,\" Feinstein said. \"There was very little=20\n",
      "willingness to help. Not even an offer of help. It is truly amazing to me,=\n",
      "=20\n",
      "because this crisis is not going to be confined to California. . . . What w=\n",
      "e=20\n",
      "really need is a period of price stability . . . and they keep going back t=\n",
      "o=20\n",
      "Nixon and price controls and saying they don't work. We all know that, but=\n",
      "=20\n",
      "what we are seeking is an interim period of stability until we can increase=\n",
      "=20\n",
      "supply.\"=20\n",
      "Tell Us What You Think -- Can you save 20 percent on your energy usage? Gov=\n",
      ".=20\n",
      "Gray Davis' administration is offering rebates to Californians who save on=\n",
      "=20\n",
      "power starting in June, and if you've got a strategy for conserving, The=20\n",
      "Chronicle wants to hear it. We'll be writing about the hardest-working ener=\n",
      "gy=20\n",
      "savers in a future story. To get involved, write to the Energy Desk, San=20\n",
      "Francisco Chronicle, 901 Mission St., San Francisco, CA 94103; or send e-ma=\n",
      "il=20\n",
      "to energysaver@sfchronicle.com.=20\n",
      "E-mail Carolyn Lochhead at clochhead@sfchronicle.com=20\n",
      ",2001 San Francisco Chronicle ? Page?A - 13=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------\n",
      "Megawatt Foolish=20\n",
      "Strike a Pose for Energy Conservation=20\n",
      "Arrol Gellner\n",
      "Wednesday, March 28, 2001=20\n",
      ",2001 San Francisco Chronicle=20\n",
      "URL:=20\n",
      "http://www.sfgate.com/cgi-bin/article.cgi?file=3D/chronicle/archive/2001/03=\n",
      "/28/H\n",
      "O179614.DTL=20\n",
      "\"Spare at the spigot,\" admonishes an old proverb, \"and let out the bunghole=\n",
      ".\"=20\n",
      "That rather tidily sums up America's haphazard attitude toward energy=20\n",
      "conservation. We gladly rally to trim our energy use by a few percent,=20\n",
      "whether at home, at work or in our cars, but we ultimately feel little=20\n",
      "urgency to change the overwhelming wastefulness of our built environment.=\n",
      "=20\n",
      "During the past two decades, we've done plenty of \"sparing at the spigot,\"=\n",
      "=20\n",
      "and that can only be applauded.=20\n",
      "We've passed minimal building energy standards such as California's Title 2=\n",
      "4=20\n",
      "but then wiped out much of the savings by building the sort of needlessly=\n",
      "=20\n",
      "bloated, energy-guzzling homes that now sprawl across acre after acre of on=\n",
      "ce=20\n",
      "pastoral landscape.=20\n",
      "We've enacted minimum standards for gas mileage -- excepting SUVs, but you=\n",
      "=20\n",
      "don't drive one of those, do you? -- yet we've made little headway in curbi=\n",
      "ng=20\n",
      "our reliance on the automobile itself.=20\n",
      "How did we get into this jam? A fair share of the blame for our disastrous=\n",
      "=20\n",
      "land-use policies belongs at the feet of the very people who insisted they=\n",
      "=20\n",
      "knew better: the postwar city planners.=20\n",
      "They've left us our current legacy of hyperorganized zoning ordinances that=\n",
      "=20\n",
      "encourage -- in fact practically mandate -- urban sprawl. These in turn hav=\n",
      "e=20\n",
      "produced a national reliance upon the automobile that has only increased.=\n",
      "=20\n",
      "ON THE ROAD AGAIN, AND AGAIN\n",
      "Flying in the face of all we've learned about the environment during the pa=\n",
      "st=20\n",
      "decades, the year 2000 set the all-time record for U.S. automobile sales,=\n",
      "=20\n",
      "with some 17.4 million cars and trucks sold.=20\n",
      "And no wonder Americans remain auto-centered. Too many planners and state=\n",
      "=20\n",
      "transportation departments still consider freeway expansion programs the=20\n",
      "solution to our mass transportation woes, even though it's been demonstrate=\n",
      "d=20\n",
      "time and again that bigger highways merely invite more traffic instead of=\n",
      "=20\n",
      "reducing it.=20\n",
      "We consumers are to blame as well, for buying into the idea that a=20\n",
      "snowballing trend of consumption is the very embodiment of success. We're=\n",
      "=20\n",
      "hooked on big houses; but because housing prices are so high in the Bay Are=\n",
      "a,=20\n",
      "we're willing to move out to less populated areas so we can afford them.=20\n",
      "We willingly drive an hour or even two to get to work -- a commute that wou=\n",
      "ld=20\n",
      "have been considered perfectly absurd even 20 years ago. Pretty soon, of=20\n",
      "course, the new community is as choked with cars and asphalt as the old one=\n",
      ".=20\n",
      "The real pity is that we've recognized the folly of these trends for decade=\n",
      "s,=20\n",
      "and we've done next to nothing to even protest them, let alone change them.=\n",
      "=20\n",
      "PLANNING FOR THE PAST\n",
      "And thanks to the hidebound attitudes of so many civic planning departments=\n",
      ",=20\n",
      "little of substance has changed in our land use policy since the 1950s: Our=\n",
      "=20\n",
      "hyperorganized zoning ordinances still jealously guard the outdated postwar=\n",
      "=20\n",
      "ideal of the single-family home surrounded by largely useless strips of=20\n",
      "\"setback\" land, and continue to frown on more intelligent arrangements such=\n",
      "=20\n",
      "as zero-lot-line construction, courtyard homes and mixed commercial and=20\n",
      "residential planning.=20\n",
      "By all means, save all the energy you can. But while any move toward=20\n",
      "conservation is commendable, it's America's fundamental building practices=\n",
      "=20\n",
      "that really need changing, not the position of the light switch in your=20\n",
      "hallway.=20\n",
      "Arrol Gellner is an Emeryville architect who also teaches at Chabot College=\n",
      ",=20\n",
      "Las Positas College and the Building Education Center. Write to him at=20\n",
      "home@sfchronicle.com.=20\n",
      ",2001 San Francisco Chronicle ? Page?WB - 1=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------\n",
      "New York mayor calls for cap on power prices=20\n",
      "\n",
      "Wednesday, March 28, 2001=20\n",
      ",2001 Associated Press=20\n",
      "URL:=20\n",
      "http://www.sfgate.com/cgi-bin/article.cgi?file=3D/news/archive/2001/03/28/n=\n",
      "ation\n",
      "al0701EST0506.DTL=20\n",
      "(03-28) 04:01 PST NEW YORK (AP) -- Mayor Rudolph Giuliani is urging federal=\n",
      "=20\n",
      "regulators to set price controls on wholesale power producers to protect=20\n",
      "consumers from rate hikes this summer.=20\n",
      "In a speech Tuesday to Wall Street and utility executives, Giuliani said a=\n",
      "=20\n",
      "surge in electricity prices last summer and an imminent shortage of power=\n",
      "=20\n",
      "supplies demand government intervention. Electricity bills rose an average =\n",
      "43=20\n",
      "percent for New Yorkers last July.=20\n",
      "``The double-digit price increases that New Yorkers had to pay last summer=\n",
      "=20\n",
      "are just unacceptable,'' Giuliani said.=20\n",
      "Giuliani said the move toward deregulation in the early 1990s was done=20\n",
      "hastily and needs to be adjusted. Until more power plants are constructed, =\n",
      "a=20\n",
      "temporary price cap would protect consumers, he said.=20\n",
      "A spokesman for KeySpan, a major New York-area energy producer, said such=\n",
      "=20\n",
      "price caps would likely discourage the construction of new power plants.=20\n",
      "In California, deregulation, power plant maintenance and scarce=20\n",
      "hydroelectricity have been blamed for an extended power crisis that has led=\n",
      "=20\n",
      "to rolling blackouts four times this year. On Tuesday, California regulator=\n",
      "s=20\n",
      "approved rate increases of up to 46 percent for customers of that state's t=\n",
      "wo=20\n",
      "largest utilities.=20\n",
      ",2001 Associated Press ?=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------\n",
      "Power regulators approve rate hikes of up to 46 percent=20\n",
      "KAREN GAUDETTE, Associated Press Writer\n",
      "Wednesday, March 28, 2001=20\n",
      ",2001 Associated Press=20\n",
      "URL:=20\n",
      "http://www.sfgate.com/cgi-bin/article.cgi?file=3D/news/archive/2001/03/28/s=\n",
      "tate0\n",
      "302EST0101.DTL=20\n",
      "(03-28) 05:53 PST SAN FRANCISCO (AP) -- Sam Sahouria owns a small downtown=\n",
      "=20\n",
      "grocery and deli, a few blocks from the offices of the state Public Utiliti=\n",
      "es=20\n",
      "Commission. He now fears a decision by the PUC could force him to lose the=\n",
      "=20\n",
      "store he has run for the past 15 years.=20\n",
      "Amid the jeers of protesters yelling ``Hell, no, we won't pay,'' the PUC=20\n",
      "voted unanimously Tuesday to approve electricity rate hikes of up to 46=20\n",
      "percent for customers of California's two largest utilities.=20\n",
      "The increases for Pacific Gas and Electric Co. and Southern California Edis=\n",
      "on=20\n",
      "Co., which take effect immediately, are the largest in California history.=\n",
      "=20\n",
      "``It's terrible, we will probably go out of business,'' Sahouria said. ``It=\n",
      "'s=20\n",
      "very simple -- we cannot afford any rate increases.''=20\n",
      "PUC commissioners, who voted 5-0 for the rate hikes, said they were necessa=\n",
      "ry=20\n",
      "to head off blackouts this summer and to keep the cash-starved utilities fr=\n",
      "om=20\n",
      "going under.=20\n",
      "However, the PUC simultaneously ordered the nearly bankrupt utilities to pa=\n",
      "y=20\n",
      "the state for billions of dollars of electricity it has bought on behalf of=\n",
      "=20\n",
      "their customers, and to pay producers of renewable energy for future=20\n",
      "electricity deliveries.=20\n",
      "``The PUC has done all it can,'' commission president Loretta Lynch said.=\n",
      "=20\n",
      "``We have fought back hard in every venue possible against these unjust=20\n",
      "energy prices.''=20\n",
      "Not all PG&E and Edison customers will face higher bills. The PUC said it=\n",
      "=20\n",
      "would create a tiered system that will protect low-income customers and=20\n",
      "penalize those customers who use the most electricity.=20\n",
      "Ratepayers have labeled the plan a ``rip-off'' that could hit as many as 10=\n",
      "=20\n",
      "million homes and businesses fighting to stay cool amid rolling blackouts a=\n",
      "nd=20\n",
      "spiraling energy costs. The rate hikes could effect as many as 25 million=\n",
      "=20\n",
      "Californians.=20\n",
      "The increase is on top of a 9 percent to 15 percent rate hike the PUC=20\n",
      "approved in January and made permanent Tuesday. An additional 10 percent=20\n",
      "increase already is scheduled for next year.=20\n",
      "``Our bills have gone from $26 to $70 for a stinking studio apartment and w=\n",
      "e=20\n",
      "don't have a heater, we use the oven to heat up the studio,'' said Belinda=\n",
      "=20\n",
      "Lazzerini, 40, who serves fruit smoothies at Jitters & Shakes in downtown S=\n",
      "an=20\n",
      "Francisco. ``The laundromat has gone up from $1.50 to $3, so now we will ha=\n",
      "ve=20\n",
      "to clean our clothes by hand and dry them in the basement. It's crazy.''=20\n",
      "Though utilities still cannot set their own rates for electricity because o=\n",
      "f=20\n",
      "a rate freeze mandated by the state's 1996 attempt at deregulation, the PUC=\n",
      "=20\n",
      "has the authority to increase rates. The rate freeze must end by March 2002=\n",
      ".=20\n",
      "The PUC meeting was disrupted at least five times by screaming protesters.=\n",
      "=20\n",
      "Before the meeting, a group led by former Green Party senatorial candidate=\n",
      "=20\n",
      "Medea Benjamin stood in the PUC chambers with yellow signs saying ``We Won'=\n",
      "t=20\n",
      "Pay.''=20\n",
      "``We don't need increases in our bills, we need increases in law enforcemen=\n",
      "t=20\n",
      "so our governor can protect us from these modern-day robber barons,'' said=\n",
      "=20\n",
      "Doug Heller of the Foundation for Taxpayer and Consumer Rights in Santa=20\n",
      "Monica.=20\n",
      "Meanwhile, the state remained free of power alerts Wednesday morning as pow=\n",
      "er=20\n",
      "reserves stayed above 7 percent.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "No explanation given after officials approve 27 percent rate hike=20\n",
      "Posted at 10:00 p.m. PST Tuesday, March 27, 2001=20\n",
      "BY STEVE JOHNSON=20\n",
      "\n",
      "Mercury News=20\n",
      "\n",
      "\n",
      "After months of waiting for the state's elected leaders to act, California'=\n",
      "s=20\n",
      "Public Utilities Commission on Tuesday unanimously approved the largest=20\n",
      "electricity rate increase in at least two decades, and offered no assurance=\n",
      "s=20\n",
      "that more hikes won't be needed soon.=20\n",
      "The 3-cent per kilowatt-hour increase would boost average rates for all=20\n",
      "customers by 27 percent, although the precise residential and business rate=\n",
      "s=20\n",
      "have yet to be determined. It is effective immediately and will show up on=\n",
      "=20\n",
      "bills in May.=20\n",
      "The commission's vote in San Francisco played out against an odd political=\n",
      "=20\n",
      "backdrop: Gov. Gray Davis, who appointed three of its five members, said he=\n",
      "=20\n",
      "wasn't consulted about the planned rate hike and that he does not favor it.=\n",
      "=20\n",
      "But Davis, after learning Sunday night about the plan, apparently did not=\n",
      "=20\n",
      "seek to block the commission's action. Instead, he issued a statement=20\n",
      "immediately after the vote that called it ``premature because we do not hav=\n",
      "e=20\n",
      "all the appropriate financial numbers necessary to make a decision. Until w=\n",
      "e=20\n",
      "do, I cannot support any rate increase.''=20\n",
      "The commission, however, said it had no alternative but to raise rates,=20\n",
      "considering that the state in recent weeks has been forced to spend billion=\n",
      "s=20\n",
      "in taxpayer funds on power because its cash-strapped utilities are no longe=\n",
      "r=20\n",
      "credit-worthy.=20\n",
      "But even some commissioners were uneasy. Davis' most recent appointee on th=\n",
      "e=20\n",
      "commission, Geoffrey Brown, acknowledged during the Tuesday meeting that ``=\n",
      "we=20\n",
      "as commissioners had very little time to review this.''=20\n",
      "In fact, an administrative law judge -- using much the same evidence that w=\n",
      "as=20\n",
      "included in the decision to raise rates -- had concluded recently that no=\n",
      "=20\n",
      "such increase was justified.=20\n",
      "The commission did not explain how it arrived at the increase of 3 cents pe=\n",
      "r=20\n",
      "kilowatt-hour. Previously, officials with Pacific Gas & Electric Co. and=20\n",
      "Southern California Edison had only sought a 2-cent per kilowatt-hour=20\n",
      "increase.=20\n",
      "Experts pointed to a number of unanswered questions that make it uncertain=\n",
      "=20\n",
      "whether a 3-cent hike will bring in sufficient revenue to cover the enormou=\n",
      "s=20\n",
      "cost the state has incurred since it began buying power.=20\n",
      "The state still doesn't know how much its effort to bail out PG&E and Ediso=\n",
      "n=20\n",
      "-- which involves a state purchase of the utilities' high-voltage lines --=\n",
      "=20\n",
      "will cost. It's also unclear how much the state might wind up spending on t=\n",
      "he=20\n",
      "expensive electricity spot market this summer, and whether an additional ra=\n",
      "te=20\n",
      "increase would be needed to cover that.=20\n",
      "``Have we got the number right?'' asked Commissioner Henry Duque, in=20\n",
      "reference to the 3-cent increase. ``I don't know. But I hope we have.''=20\n",
      "Roderick Wright, D-Los Angeles, who chairs the Assembly Utilities Committee=\n",
      ",=20\n",
      "told the commission that he opposed the idea of approving any rate hike now=\n",
      ",=20\n",
      "until the state has a better idea of the total amount that consumers=20\n",
      "ultimately will have to bear.=20\n",
      "By establishing a 3-cent increase now, ``we'll be back in a short time,=20\n",
      "potentially doing it again,'' he said. ``To be nickeled and dimed to death =\n",
      "as=20\n",
      "we're doing here, I don't think that does anybody any good.''=20\n",
      "Signs of confusion=20\n",
      "Utility officials sounded similarly unimpressed and confused.=20\n",
      "``While providing a welcome dose of realism,'' PG&E president Gordon Smith=\n",
      "=20\n",
      "said in a prepared statement, the rate hike and other actions taken by the=\n",
      "=20\n",
      "commission Tuesday ``do not offer a comprehensive solution, fail to resolve=\n",
      "=20\n",
      "the uncertainty of the crisis and may even create more instability.''=20\n",
      "Among PG&E's major concerns is whether the utilities will be compensated fo=\n",
      "r=20\n",
      "their debts -- which they claim total about $13 billion -- and whether PG&E=\n",
      "=20\n",
      "or the state will be responsible for the cost of power purchased on the spo=\n",
      "t=20\n",
      "market.=20\n",
      "Bruce Foster, an Edison vice president, offered a similar lukewarm=20\n",
      "endorsement of the increase. ``I'm not saying that it will do it,'' he said=\n",
      ",=20\n",
      "acknowledging the possibility of more increases down the road, but he calle=\n",
      "d=20\n",
      "the hike ``a step in the right direction.''=20\n",
      "Rate hike vs. bankruptcy=20\n",
      "Commissioner Richard Bilas said he feared that if the utilities were forced=\n",
      "=20\n",
      "into bankruptcy, the judge overseeing such a case could order much bigger=\n",
      "=20\n",
      "rate hikes to stabilize the companies' finances.=20\n",
      "``A rate increase designed by this commission is far better than one design=\n",
      "ed=20\n",
      "by a bankruptcy judge,'' Bilas said, adding that he also was concerned abou=\n",
      "t=20\n",
      "how the state appeared to be buying electricity ``like a drunken sailor.''=\n",
      "=20\n",
      "But others reacted angrily to the commission's decision. Several protesters=\n",
      "=20\n",
      "hoisted signs at the meeting that read, ``we won't pay'' and chanted, ``rat=\n",
      "e=20\n",
      "hikes go away, make the energy companies pay.'' Police later escorted them=\n",
      "=20\n",
      "out of the meeting after commission President Loretta Lynch became annoyed =\n",
      "at=20\n",
      "their noisy interruptions.=20\n",
      "Instead of raising rates, they and other consumer advocates urged Davis to=\n",
      "=20\n",
      "use the state's power of eminent domain to take over the power plants=20\n",
      "operated by companies accused of gouging the public.=20\n",
      "``The governor should start seizing plants until these crooks start playing=\n",
      "=20\n",
      "by the rules of society,'' said Mike Florio, senior attorney with The Utili=\n",
      "ty=20\n",
      "Reform Network in San Francisco. ``They're economic war criminals.''=20\n",
      "Officials in Davis' office said Tuesday that they intend to review the=20\n",
      "commission's action and decide over the next few weeks whether to support i=\n",
      "t=20\n",
      "or seek changes. But to some people, the governor's claim of ignorance abou=\n",
      "t=20\n",
      "the commission's plan -- and his seeming inability to do anything to stop t=\n",
      "he=20\n",
      "rate hike from being approved -- severely strained credulity.=20\n",
      "``Give me a break,'' said Sen. John Vasconcellos, D-San Jose. ``Who appoint=\n",
      "ed=20\n",
      "them and who's been giving orders all along?''=20\n",
      "Davis did say for the first time Tuesday that he might be willing to consid=\n",
      "er=20\n",
      "having consumers pay more for electricity. ``If it becomes clear that a rat=\n",
      "e=20\n",
      "increase is absolutely necessary for the good of the state,'' he said, ``I=\n",
      "=20\n",
      "will support one that is fair.''=20\n",
      "\n",
      "\n",
      "Dion Nissenbaum and Hallye Jordan contributed to this report.\n",
      "\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "--------------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Impact of increases will vary greatly=20\n",
      "Posted at 9:44 p.m. PST Tuesday, March 27, 2001=20\n",
      "BY JOHN WOOLFOLK=20\n",
      "\n",
      "Mercury News=20\n",
      "\n",
      "\n",
      "The rate increases approved Tuesday will hit Californians in very different=\n",
      "=20\n",
      "ways: The heaviest users of electricity will probably pay the most, and=20\n",
      "nearly half of residential users will likely see no increase at all.=20\n",
      "Although the details aren't yet final, state regulators say they intend to=\n",
      "=20\n",
      "fashion a ``tiered structure'' to implement the higher rates. Here's how th=\n",
      "e=20\n",
      "proposal will affect your bill:=20\n",
      "When do the higher rates kick in?=20\n",
      "Regulators consider the increase effective immediately but say it won't=20\n",
      "appear on bills until May. While regulators agreed that rates will rise, th=\n",
      "ey=20\n",
      "still have to approve a structure for how customers will share the cost.=20\n",
      "Hearings are expected next month. Once the structure is approved, regulator=\n",
      "s=20\n",
      "say, utilities can apply it retroactively to Tuesday's rate approval.=20\n",
      "Do the new rates affect all Californians?=20\n",
      "No. The Public Utilities Commission, which approved the rate increase, has=\n",
      "=20\n",
      "authority over only investor-owned utilities: Pacific Gas & Electric Co.,=\n",
      "=20\n",
      "Southern California Edison and San Diego Gas & Electric, which together ser=\n",
      "ve=20\n",
      "about 10 million homes and businesses. The increase affects PG&E and Edison=\n",
      ",=20\n",
      "but not San Diego. Municipal utilities, including Santa Clara, Palo Alto,=\n",
      "=20\n",
      "Alameda, Sacramento and Los Angeles are governed independently and exempt.=\n",
      "=20\n",
      "How will the new rates work?=20\n",
      "Regulators are planning a tiered rate structure in which large commercial a=\n",
      "nd=20\n",
      "power-guzzling residential customers pay a greater share. Residential=20\n",
      "customers would pay more according to how much they exceed their ``baseline=\n",
      "''=20\n",
      "amount.=20\n",
      "What is the baseline?=20\n",
      "The baseline is considered the average household usage. It was established=\n",
      "=20\n",
      "years ago to encourage conservation by charging a higher rate for electrici=\n",
      "ty=20\n",
      "usage over that amount.=20\n",
      "Does everybody have the same baseline?=20\n",
      "No. Customers are assigned different baselines to account for variations in=\n",
      "=20\n",
      "electricity needs caused by regional climate, household type and season.=20\n",
      "How many baselines are there?=20\n",
      "PG&E residential customers in the San Francisco Bay Area are divided among=\n",
      "=20\n",
      "three climatic baseline territories: the coastal areas including San=20\n",
      "Francisco and the Peninsula, the East Bay and South Bay, and the Santa Cruz=\n",
      "=20\n",
      "Mountains.=20\n",
      "Within those territories, there are separate baselines for single-family=20\n",
      "homes and multifamily apartment complexes, as well as for homes with only=\n",
      "=20\n",
      "electric appliances instead of both gas and electric.=20\n",
      "In addition, baseline rates are adjusted for summer and winter usage. Summe=\n",
      "r=20\n",
      "rates begin May 1, and winter rates begin Nov. 1.=20\n",
      "How do I find out my baseline?=20\n",
      "Your electric bill shows the baseline amount in kilowatt-hours and the rate=\n",
      "=20\n",
      "you are billed for that amount. It also shows how much power you used over=\n",
      "=20\n",
      "the baseline and the rate you were billed for it.=20\n",
      "How does the baseline affect my bill?=20\n",
      "Customers already pay about 14 percent more for power they use over their=\n",
      "=20\n",
      "baseline. Under the proposed tiered rates, PG&E customers who use no more=\n",
      "=20\n",
      "than 130 percent of their baseline amount wouldn't pay any more than they d=\n",
      "o=20\n",
      "now. PG&E customers who use up to twice their baseline would see an average=\n",
      " 9=20\n",
      "percent increase in monthly bills. Those who use more than twice their=20\n",
      "baseline would see an average increase of 36 percent. Edison customers woul=\n",
      "d=20\n",
      "have similar rates. Almost half of the utilities' customers don't exceed 13=\n",
      "0=20\n",
      "percent of their baseline.=20\n",
      "Why am I hearing different reports about the size of the rate increase?=20\n",
      "The increase can be calculated many ways. The commission approved an increa=\n",
      "se=20\n",
      "of 3 cents a kilowatt-hour, three times the 1-cent surcharge approved in=20\n",
      "January, which was made permanent Tuesday. Because the 1-cent increase=20\n",
      "averaged 10 percent, some have said that, plus the 3 cents, amounts to a=20\n",
      "40-percent increase.=20\n",
      "Others have added 3 cents to the amount the utilities charge just for=20\n",
      "electricity, which for PG&E is now 6.4 cents, and come up with a 47-percent=\n",
      "=20\n",
      "increase.=20\n",
      "Because total rates include transmission and maintenance surcharges, others=\n",
      "=20\n",
      "have added 3 cents to the average total rate of 11.14 cents for all=20\n",
      "customers, a 27-percent increase. Under the tiered proposal, the average=20\n",
      "residential rate would rise 20.8 percent from 12.46 cents to 15.05 cents.=\n",
      "=20\n",
      "But average bills for residents' bills would increase up to 36 percent,=20\n",
      "depending on electricity use.=20\n",
      "Can my business avoid the rate hike by saving energy?=20\n",
      "The proposed tiered rate structure based on the baseline use doesn't apply =\n",
      "to=20\n",
      "commercial and industrial businesses. In fact, the proposal calls for a=20\n",
      "proportionately greater increase for businesses, which pay lower rates than=\n",
      "=20\n",
      "residential customers. Commercial customers would see a 22.6-percent=20\n",
      "increase. Industrial customers, generally over 500 kilowatt-hours a month,=\n",
      "=20\n",
      "would see a 41-percent increase.=20\n",
      "But businesses can still get a discount on summer bills by saving power. Go=\n",
      "v.=20\n",
      "Gray Davis this month ordered 20-percent rebates for all customers who redu=\n",
      "ce=20\n",
      "electricity usage 20 percent over last year between June and September.=20\n",
      "Residential and small commercial rebates will be based on 20 percent=20\n",
      "reduction of total consumption, while commercial and industrial rebates wil=\n",
      "l=20\n",
      "be based on 20 percent reduction of peak load.=20\n",
      "If I qualify for assistance in paying my bills now, will I have to pay more=\n",
      "?=20\n",
      "Possibly. The proposal calls for exempting customers in the California=20\n",
      "Alternative Rates for Energy program, and expanding eligibility from 150=20\n",
      "percent to 175 percent of federal poverty guidelines. But commissioners=20\n",
      "aren't ruling out tiered rates for those customers to encourage them to sav=\n",
      "e=20\n",
      "electricity.=20\n",
      "\n",
      "\n",
      "Contact John Woolfolk at (408) 278-3410 or at jwoolfolk@sjmercury.com=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "PUC raises power rates=20\n",
      "Edison customers could see monthly bills go up by 27%. The agency plans to=\n",
      "=20\n",
      "build a tiered system costing more for heavy users.=20\n",
      "March 28, 2001=20\n",
      "From Register staff=20\n",
      "and news service reports=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Dorothy Kascak, 82, of Cypress holds her dog, Baron. Kascak needs oxygen fu=\n",
      "ll=20\n",
      "time. She's afraid power rate hikes will make air conditioning too costly\n",
      "Photo: Bruce Chambers / The Register\n",
      "?\n",
      "?\n",
      "\n",
      "SAN FRANCISCO Amid the jeers of protesters yelling \"Hell no, we won't pay,\"=\n",
      "=20\n",
      "the state Public Utilities Commission voted unanimously Tuesday to approve=\n",
      "=20\n",
      "electricity rate increases of as high as 36 percent for customers of=20\n",
      "California's two largest power utilities.=20\n",
      "The rate increases for Pacific Gas & Electric Co. and Southern California=\n",
      "=20\n",
      "Edison Co., which take effect immediately, are the largest in California=20\n",
      "history, totaling $4.8 billion a year.=20\n",
      "PUC commissioners said the increases were necessary to head off blackouts=\n",
      "=20\n",
      "this summer and to keep the cash-starved utilities from going under.=20\n",
      "The PUC simultaneously ordered the utilities to pay the state for billions =\n",
      "of=20\n",
      "dollars of electricity it has bought on behalf of their customers, and to p=\n",
      "ay=20\n",
      "producers of renewable energy for future supplies.=20\n",
      "Commission President Loretta Lynch said the PUC had tried to avoid rate=20\n",
      "increases but saw no other option.=20\n",
      "The commission voted to boost rates by 3 cents a kilowatt-hour. PG&E rates=\n",
      "=20\n",
      "would rise by as much as 36 percent and Edison rates by as much as 27=20\n",
      "percent.=20\n",
      "Not all PG&E and Edison customers will face higher bills.=20\n",
      "The PUC said it would create a tiered system that will protect customers wh=\n",
      "o=20\n",
      "don't use much electricity and penalize those who use more.=20\n",
      "Dorothy Kascak, 82, of Cypress is on 24-hour oxygen supplied by a machine i=\n",
      "n=20\n",
      "her home that runs on electricity.=20\n",
      "Her bill is already $135 a month despite the fact that she gets a reduction=\n",
      "=20\n",
      "for being on life support.=20\n",
      "\"They say we're going to have to go without air conditioning in the summer,=\n",
      "\"=20\n",
      "she said.=20\n",
      "\"I can't live without air conditioning. I have a bad heart, bad lungs and I=\n",
      "'m=20\n",
      "a diabetic.\"\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Energy payments not enough, gas-using generators say=20\n",
      "March 28, 2001=20\n",
      "By HANH KIM QUACH\n",
      "The Orange County Register=20\n",
      "Natural-gas-dependent alternative generators say that even if the major=20\n",
      "utilities begin paying them for energy, it won't be enough cash.=20\n",
      "State regulators at the Public Utilities Commission on Tuesday ordered=20\n",
      "Southern California Edison and Pacific Gas & Electric to start paying=20\n",
      "alternative-energy producers $79 per megawatt-hour. But alternative=20\n",
      "generators that use natural gas say that's at least $20 below what it costs=\n",
      "=20\n",
      "to produce the energy.=20\n",
      "\"On the short run, we can't generate,'' said David Lloyd of NRG Energy Inc.=\n",
      ",=20\n",
      "which this month shut down its 260- megawatt plant in the Bay area that sel=\n",
      "ls=20\n",
      "steam to the C&H sugar factory next door. \"It just costs so much for=20\n",
      "electricity.''=20\n",
      "Alternative generators have been paid little, or nothing, the past four=20\n",
      "months and are now collectively owed more than $1 billion by the utilities.=\n",
      "=20\n",
      "The situation has prompted many of the small generators to scale back or sh=\n",
      "ut=20\n",
      "down their plants, resulting in a 2,900-megawatt loss to California - enoug=\n",
      "h=20\n",
      "to power 2.9 million homes. That was a factor in blackouts last week and th=\n",
      "e=20\n",
      "state's Stage 2 energy alert Tuesday.=20\n",
      "There are two classes of alternative generators: renewables, which use sola=\n",
      "r,=20\n",
      "wind or clean-burning technologies to produce energy, and co-generators,=20\n",
      "which use natural gas to produce energy but share the heat output with a=20\n",
      "manufacturer. Renewables, which make up half of the 10,000 alternative-ener=\n",
      "gy=20\n",
      "megawatts, say they're likely to continue producing but would like to be pa=\n",
      "id=20\n",
      "what they've been owed the past four months.=20\n",
      "PUC spokeswoman Kyle Devine said $79 per megawatt-hour should work for=20\n",
      "generators if the natural gas is transported through an Oregon pipeline - a=\n",
      "=20\n",
      "cheaper route - rather than through a pipeline from the Southwest.=20\n",
      "But a gas proposal that Ken Ross of Delta Energy Co. in Santa Ana received=\n",
      "=20\n",
      "Tuesday showed the transportation price to be just as expensive.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "---------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Bush to nominate 2 who favor free markets for energy panel=20\n",
      "March 28, 2001=20\n",
      "By DENA BUNIS\n",
      "The Orange County Register=20\n",
      "WASHINGTON President George W. Bush will nominate two new federal energy=20\n",
      "regulators whose history indicates that, like him, they favor free markets=\n",
      "=20\n",
      "and don't favor price controls.=20\n",
      "Bush said Tuesday that he will ask the Senate to confirm Patrick Wood,=20\n",
      "chairman of the Texas Public Utility Commission, and Nora Mead Brownell, a=\n",
      "=20\n",
      "commissioner with the Pennsylvania PUC, to the five-member Federal Energy=\n",
      "=20\n",
      "Regulatory Commission.=20\n",
      "Both Republican nominees have presided over the deregulation of their state=\n",
      "'s=20\n",
      "energy markets, so far with more success than California.=20\n",
      "Sen. Dianne Feinstein, D-Calif., who sits on the Energy and Natural Resourc=\n",
      "es=20\n",
      "Committee, which will hold hearings on the FERC appointees, issued an early=\n",
      "=20\n",
      "warning.=20\n",
      "\"Certainly my vote would be dependent on whether they would be responsive t=\n",
      "o=20\n",
      "the present situation,'' Feinstein said. \"Up to this point, the=20\n",
      "administration has been recalcitrant to the point of arrogance. If that is=\n",
      "=20\n",
      "the case with these new appointees as well, they won't have my vote.\"=20\n",
      "However, neither is expected to have trouble getting confirmation.=20\n",
      "Wood, PUC chairman since 1995, has said Texas' deregulation climate is=20\n",
      "different from California's.=20\n",
      "Unlike California, the Texas deregulation encourages long-term contracts.=\n",
      "=20\n",
      "Texas has more power reserves and it's easier to get a permit to build a=20\n",
      "plant there.=20\n",
      "Brownell has been on the Pennsylvania PUC since 1997.=20\n",
      "Pennsylvania's market also differs greatly from California's in several way=\n",
      "s.=20\n",
      "The state is a net exporter of power while California is an importer, and=\n",
      "=20\n",
      "Pennsylvania's deregulation plan did not force utilities to sell their=20\n",
      "plants.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "----------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Davis backs power rate hikes if 'necessary'=20\n",
      "He says PUC approval of hikes Tuesday is 'premature.' S&P is unsure how=20\n",
      "conservation will affect revenue.=20\n",
      "March 28, 2001=20\n",
      "By JOHN HOWARD\n",
      "The Orange County Register=20\n",
      "SACRAMENTO Gov. Gray Davis shifted his position opposing electricity rate=\n",
      "=20\n",
      "hikes, saying he would support them \"if it becomes clear it is absolutely=\n",
      "=20\n",
      "necessary for the good of the state.\"=20\n",
      "The governor's statement Tuesday came just hours after the Public Utilities=\n",
      "=20\n",
      "Commission, a panel controlled by Davis' appointees, endorsed a plan to rai=\n",
      "se=20\n",
      "monthly bills by as much 36 percent. The increase eventually would be tied =\n",
      "to=20\n",
      "a flexible rating plan in which light users will pay a little more, medium=\n",
      "=20\n",
      "users (those who conserve energy) will pay moderately more, and heavy users=\n",
      "=20\n",
      "will pay a lot more. Details of the plan are not yet set.=20\n",
      "Davis called the PUC's decision \"premature,\" saying scant information was=\n",
      "=20\n",
      "available to justify increases.=20\n",
      "\"While I have opposed rate increases, if it becomes clear a rate increase i=\n",
      "s=20\n",
      "absolutely necessary for the good of the state, I will support one that is=\n",
      "=20\n",
      "fair and do my duty to convince Californians of its necessity,\" Davis said.=\n",
      "=20\n",
      "Boosting electricity rates to help resolve California's energy crisis is=20\n",
      "likely to increase the state's $10 billion revenue bond sale to $12 billion=\n",
      "=20\n",
      "or more, entailing more interest charges and a longer payback.=20\n",
      "Standard and Poor's, the Wall Street credit-rating agency, called the ratin=\n",
      "g=20\n",
      "plan \"a complicated formula that will only increase rates on heavy users of=\n",
      "=20\n",
      "electricity.\"=20\n",
      "\"It is unclear exactly how many users will be affected, and whether revenue=\n",
      "s=20\n",
      "will slip under the rate-raising formula as consumers respond by conserving=\n",
      "=20\n",
      "electricity,\" S&P said.=20\n",
      "Also unclear is how the money from increased rates would be divided. The=20\n",
      "state, which has been buying power on behalf of the utilities, says it has=\n",
      "=20\n",
      "first rights to the money. The utilities have challenged that.=20\n",
      "California hopes to borrow $10 billion by selling revenue bonds in May to=\n",
      "=20\n",
      "cover its costs of purchasing emergency electricity since December. Through=\n",
      "=20\n",
      "April 7, those costs are expected to total $4.2 billion.=20\n",
      "The bond money also will be used to pay for power under long-term contracts=\n",
      ",=20\n",
      "which is much cheaper than power bought just before it is used.=20\n",
      "Critics have questioned whether $10 billion will be sufficient. PUC Preside=\n",
      "nt=20\n",
      "Loretta Lynch said the increased rates would support at least a $12 billion=\n",
      "=20\n",
      "bond sale. Davis spokesman Steve Maviglio and Joseph Fichera, a Davis fisca=\n",
      "l=20\n",
      "adviser, declined to discuss a specific amount.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "--------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Cheney says state's on its own=20\n",
      "March 28, 2001=20\n",
      "By DENA BUNIS\n",
      "The Orange County Register=20\n",
      "WASHINGTON Sen. Dianne Feinstein says Vice President Dick Cheney on Tuesday=\n",
      "=20\n",
      "offered no hope of federal help for California's electricity crisis, and sh=\n",
      "e=20\n",
      "is once again appealing to the president to meet with her.=20\n",
      "\"It was very disappointing,'' Feinstein said in an interview after a=20\n",
      "45-minute meeting Cheney had with a group of 10 senators. \"I practically=20\n",
      "begged for a period of reliability and stability so we could get through th=\n",
      "e=20\n",
      "summer, and I didn't get any feeling back that there was going to be any=20\n",
      "help.''=20\n",
      "Cheney insisted, Feinstein said, that California has to \"straighten itself=\n",
      "=20\n",
      "out'' and was not moved by the fact that the Public Utility Commission was=\n",
      "=20\n",
      "poised to increase rates nearly 40 percent.=20\n",
      "After the meeting, Feinstein sent President George W. Bush a two-page lette=\n",
      "r.=20\n",
      "\"Your administration has not yet offered any constructive solution to this=\n",
      "=20\n",
      "problem,'' she wrote. \"If the federal government continues to ignore the=20\n",
      "situation, the result may very well be economic disaster for millions of=20\n",
      "small businesses and residents.''=20\n",
      "Feinstein wrote a final plea that Bush meet with her and Sen. Gordon Smith,=\n",
      "=20\n",
      "R-Ore., who is co-sponsoring a bill authorizing short-term regional price=\n",
      "=20\n",
      "caps.=20\n",
      "Bush on Tuesday reiterated his position against price caps in a speech in=\n",
      "=20\n",
      "Michigan.=20\n",
      "Feinstein has heard little sympathy for her state from colleagues, who have=\n",
      "=20\n",
      "repeatedly said Californians have not felt any real pain. Rep. Jennifer Dun=\n",
      "n,=20\n",
      "R-Wash., recently complained that Californians were turning on their hot tu=\n",
      "bs=20\n",
      "without worrying about paying higher costs.=20\n",
      "Feinstein said such criticism is exaggerated. But she said she hopes the ra=\n",
      "te=20\n",
      "increase approved by the PUC will show critics the state \"is doing the righ=\n",
      "t=20\n",
      "thing.''=20\n",
      "Sen. Barbara Boxer, D-Calif., who said she was not invited to meet with=20\n",
      "Cheney, said she was concerned that the rate increase would be \"making=20\n",
      "consumers bear the brunt\" of the electricity crisis.=20\n",
      "Before a rate increase is considered, Boxer said, the PUC should make sure=\n",
      "=20\n",
      "average residential consumers and small businesses are protected and take=\n",
      "=20\n",
      "whatever legal action is available against \"those generating companies who=\n",
      "=20\n",
      "colluded and unlawfully manipulated prices.=20\n",
      "\"We should get something in return, some assets,\" Boxer added, also calling=\n",
      "=20\n",
      "on the utilities to contribute some of the money they diverted to their=20\n",
      "parent companies.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "------------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Highest rate hike in state history raises consumer anger=20\n",
      "O.C. residents and businesses resent 'energy hog' label.=20\n",
      "March 28, 2001=20\n",
      "By KATE BERRY\n",
      "The Orange County Register=20\n",
      "Helen Lewis, a Villa Park homemaker, believes the rate hike unfairly target=\n",
      "s=20\n",
      "families with children.=20\n",
      "Dorothy Kascak, an 82-year-old Cypress resident who uses an oxygen machine=\n",
      "=20\n",
      "24-hours a day, is furious at being called an \"energy hog.''=20\n",
      "And Vic Peterson, a homebuilder in Corona del Mar, has angry words for Gov.=\n",
      "=20\n",
      "Gray Davis for allowing the energy crisis to escalate, potentially=20\n",
      "endangering the economy.=20\n",
      "Throughout Orange County, residents and businesses responded angrily to the=\n",
      "=20\n",
      "approval Tuesday of a 3-cent-a-kilowatt-hour rate increase by the Californi=\n",
      "a=20\n",
      "Public Utilities Commission.=20\n",
      "\"There are a lot of handicapped citizens that are on fixed incomes,'' said=\n",
      "=20\n",
      "Kascak, who has been on an oxygen machine for eight years. \"If they raise o=\n",
      "ur=20\n",
      "rates 40 percent, what am I going to do?''=20\n",
      "Though oxygen users typically receive a discount on their electricity rates=\n",
      ",=20\n",
      "they will be subject to the rate increase like everyone else, said Bruce=20\n",
      "Foster, vice president of regulatory affairs at Southern California Edison.=\n",
      "=20\n",
      "Many businesses are concerned that higher power costs could push California=\n",
      "=20\n",
      "and the nation into a recession.=20\n",
      "\"It's upsetting,'' said Peterson, who owns Peterson Construction & Design=\n",
      "=20\n",
      "Inc. \"Less money coming into California means less people purchasing. Who a=\n",
      "re=20\n",
      "we going to sell homes to?''=20\n",
      "At Panama Joe's Hair Design in Irvine, owner Joe Gayle said he has no choic=\n",
      "e=20\n",
      "but to pay the higher rates. He's got hair dryers, electric hair clippers,=\n",
      "=20\n",
      "lights, a refrigerator, a microwave and computer to run a nine-station hair=\n",
      "=20\n",
      "salon.=20\n",
      "Last month Gayle's bill was $234. A year ago it was $183. Now he expects it=\n",
      "=20\n",
      "to top $300 a month.=20\n",
      "\"They jack it up because they know there is nothing we can do,\" he said.=20\n",
      "Then there's the effect on families, especially those who live in large=20\n",
      "houses.=20\n",
      "The PUC will debate in the next month how to implement a tiered-rate plan,=\n",
      "=20\n",
      "with large users paying the highest rates. A plan proposed Monday by PUC=20\n",
      "President Loretta Lynch targets what she called \"energy hogs\" by increasing=\n",
      "=20\n",
      "rates on usage over 130 percent of a consumer's minimum \"baseline\"=20\n",
      "allocation, which varies by neighborhood.=20\n",
      "For Lewis, the Villa Park homemaker, conservation will barely put a dent in=\n",
      "=20\n",
      "her electric bill. The 4,300-square-foot house she shares with her husband=\n",
      "=20\n",
      "and three children has a monthly baseline of 267 kilowatt-hours, while the=\n",
      "=20\n",
      "family's energy consumption routinely runs from 1,200 to 1,500=20\n",
      "kilowatts-hours.=20\n",
      "\"It's not as if we're leaving the lights on,'' said Lewis. \"But they are=20\n",
      "taking people that have houses and have washers and dryers, and penalizing=\n",
      "=20\n",
      "them. Families will be hit very hard.''=20\n",
      "Register staff writer Elizabeth Aguilera contributed to this report.=20\n",
      "\n",
      "\n",
      "\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Lighten up ... it's just a California thing=20\n",
      "Outside the state, there is almost a smug satisfaction in our energy=20\n",
      "failures. But most here remain optimistic.=20\n",
      "March 28, 2001=20\n",
      "By TERI SFORZA\n",
      "The Orange County Register=20\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Beckie Boyden of Irvine Orthopedics and Physical Medicine tends to her=20\n",
      "receptionist duties by candlelight during a recent rolling blackout\n",
      "Photo: Bruce Chambers / The Register\n",
      "?\n",
      "?\n",
      "\n",
      "California. Global trendsetter. High-tech nexus. Crucible for creativity.=\n",
      "=20\n",
      "The Golden State is raw fiscal muscle, with an economy bigger than China's=\n",
      "=20\n",
      "and twice the size of Canada's. But alas. California can't keep the lights=\n",
      "=20\n",
      "on.=20\n",
      "\"It's amazing to think that we have the world's seventh-largest economy - a=\n",
      "nd=20\n",
      "a Third World power system,\" said Fred Smoller, chairman of the=20\n",
      "political-science department at Chapman University. \"It's certainly a blow =\n",
      "to=20\n",
      "our collective ego.\"=20\n",
      "Rolling blackouts finally hit Orange County last week. Workers were plunged=\n",
      "=20\n",
      "into darkness. Children studied by candlelight. Cars crawled through lawles=\n",
      "s=20\n",
      "intersections. Elevators halted between floors, libraries checked out books=\n",
      "=20\n",
      "by hand, and portable generators were rushed to sewer-lift stations to keep=\n",
      "=20\n",
      "raw sewage from spilling into the ocean.=20\n",
      "Outsiders scoff. Californians are a gaggle of goofy tree-huggers guilty of=\n",
      "=20\n",
      "\"mass myopia\" and \"willful energy abnegation,\" charged Charles Krauthammer,=\n",
      "=20\n",
      "columnist for the New York Daily News.=20\n",
      "\"Californians refuse to acknowledge that in the real world, their desire fo=\n",
      "r=20\n",
      "one good (an unsullied environment) might actually conflict with another=20\n",
      "desire (hot water in their Jacuzzis).\"=20\n",
      "Sensing weakness, other states are pouncing.=20\n",
      "Glow-in-the-dark mouse pads saying \"Michigan. We never leave you in the dar=\n",
      "k\"=20\n",
      "are landing in the mailboxes of California CEOs. Alongside flashlights=20\n",
      "boasting \"The lights are always on in Tennessee.\" And nine-volt batteries,=\n",
      "=20\n",
      "courtesy of the Greater Raleigh (N.C.) Chamber of Commerce (\"In the dark=20\n",
      "about where to grow your business?\").=20\n",
      "\"Our CEO got the flashlight pitch,\" said David Sonksen, executive vice=20\n",
      "president and chief financial officer of Microsemi Corp. in Santa Ana. \"But=\n",
      "=20\n",
      "he threw it away. It would take 10 years of exorbitant power costs to even=\n",
      "=20\n",
      "make a dent in the costs associated with moving.\"=20\n",
      "Clever ambulance chasing, Orange County Business Council Vice President Bil=\n",
      "l=20\n",
      "Carney calls it. And while some companies say they'll think seriously about=\n",
      "=20\n",
      "expanding, it doesn't seem like anyone intends to leave.=20\n",
      "\"California has it all over everywhere else. Sorry!\" said Lisa Green, a vic=\n",
      "e=20\n",
      "president for Entridia Corp. \"The power thing is a little bit of a pain, bu=\n",
      "t=20\n",
      "not that much of a pain.\"=20\n",
      "\"I'll take sweltering on a summer day over a Michigan winter any time,\" sai=\n",
      "d=20\n",
      "Laer Pearce, a public relations consultant in Laguna Hills.=20\n",
      "Which is not to dismiss the seriousness of the crisis. \"I'm concerned about=\n",
      "=20\n",
      "what it means for our economy,\" said Naomi Vine, director of the Orange=20\n",
      "County Museum of Art, whose offices went dark without warning one day last=\n",
      "=20\n",
      "week. \"If businesses can't be productive, what does that mean for the year=\n",
      "=20\n",
      "ahead?\"=20\n",
      "\"It's really scary,\" said Pearce, who held his breath as the lights went ou=\n",
      "t=20\n",
      "in buildings across the street last week. \"This is one that, when they were=\n",
      "=20\n",
      "studying California's infrastructure, nobody even predicted. What's going t=\n",
      "o=20\n",
      "happen with the sewers and roads and airports and everything else?\"=20\n",
      "Indeed, the crisis may be exposing the weak spots in California's go-go=20\n",
      "economy.=20\n",
      "One concern is that production shifted this year from California to more=20\n",
      "energy-reliable areas might never return, Carney said. And if, even for the=\n",
      "=20\n",
      "short term, California businesses are perceived as being unreliable supplie=\n",
      "rs=20\n",
      "because they have unreliable power, the effects could be lasting, he said.=\n",
      "=20\n",
      "Others worry about the effect of outages on people themselves. As a culture=\n",
      ",=20\n",
      "we've becoming utterly dependent upon electricity to work, to play, to=20\n",
      "communicate. \"Everything we do is electric,\" said Pearce. \"We just couldn't=\n",
      "=20\n",
      "go back to knocking it out on a manual typewriter anymore.\"=20\n",
      "And it's starting to sink in that things will get worse before they get=20\n",
      "better, said Al Milo, director at the Fullerton Public Library, which lost=\n",
      "=20\n",
      "power briefly last week.=20\n",
      "But things will get better, the stubborn optimists say.=20\n",
      "\"This is a great thing that's happening,\" said Gerran Brown of Brown Coloni=\n",
      "al=20\n",
      "Mortuary in Santa Ana. \"Energy prices may go up temporarily, but people wil=\n",
      "l=20\n",
      "learn to turn lights off and not drive gas-guzzling SUVs, and eventually=20\n",
      "we'll have a truly free market where the consumer will win.\"=20\n",
      "California will emerge from this darkness stronger than ever, with new=20\n",
      "technologies, a new conservation ethic and a new emphasis on alternative,=\n",
      "=20\n",
      "renewable forms of energy. \"It's a new opportunity for us to be an=20\n",
      "international leader,\" Carney said.=20\n",
      "Carney was hosting two Korean businessmen last week when there was a great,=\n",
      "=20\n",
      "sudden click and lights went out at the Business Council's office. \"They to=\n",
      "ld=20\n",
      "me politely, to make me feel better I'm sure, that they have blackouts in=\n",
      "=20\n",
      "Korea, too,\" Carney said.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "----------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Conserve and save=20\n",
      "A proposed tiered-billing format is designed to spare low power users from=\n",
      "=20\n",
      "the rate increases. Baseline is the bottom line.=20\n",
      "March 28, 2001=20\n",
      "By ANNE C. MULKERN\n",
      "and KATE BERRY\n",
      "The Orange County Register=20\n",
      "The state's power crisis hit your wallet Tuesday when state regulators hike=\n",
      "d=20\n",
      "electricity rates as much as 36 percent.=20\n",
      "Your bill could be headed up as a result. Here are answers to some of your=\n",
      "=20\n",
      "questions about the rate hike:=20\n",
      "Q: How much more will I pay?=20\n",
      "A: Details of the plan haven't been finalized. But the idea is to create a=\n",
      "=20\n",
      "three-level rate structure. Those who use more electricity will pay the=20\n",
      "highest rates and those who conserve will save.=20\n",
      "Consumers who use less than l30 percent of their baseline power allocation=\n",
      "=20\n",
      "will escape any increase. (We explain baseline below.)=20\n",
      "The rate increase will be distributed across the two higher tiers. The Publ=\n",
      "ic=20\n",
      "Utilities Commission over the next 30 days will come up with a plan to=20\n",
      "determine exact rates for the two higher levels, as well as for different=\n",
      "=20\n",
      "types of customers, such as residential and small business.=20\n",
      "Q: So what is baseline?=20\n",
      "A: Baseline is the portion of your usage considered necessary for essential=\n",
      "s.=20\n",
      "To encourage conservation, it's set below the amount most people use.=20\n",
      "The percentage of Edison customers who use less than 130 percent of baselin=\n",
      "e=20\n",
      "is roughly 48 percent in winter months and 45 percent in the summer.=20\n",
      "Because baseline is set based on the average household's use, it means a=20\n",
      "single person living in an apartment can have the same baseline as a=20\n",
      "homeowner with four children.=20\n",
      "Baseline varies depending on where you live. Warmer areas get higher=20\n",
      "baselines. There are different baselines for the summer and winter. People=\n",
      "=20\n",
      "who use medical equipment in their homes can get higher baseline allocation=\n",
      "s.=20\n",
      "All-electric homes get higher baselines.=20\n",
      "Q: What's the reason for this rate hike? Are they really doing this to get=\n",
      "=20\n",
      "people to conserve?=20\n",
      "A: Conservation is a side benefit. The primary motivation is money.=20\n",
      "The state has already shelled out $3 billion buying power in the last three=\n",
      "=20\n",
      "months for customers of the near-insolvent utilities.=20\n",
      "The state plans to issue bonds, which will be paid for by consumers, to cov=\n",
      "er=20\n",
      "the power purchases. But Wall Street bankers must have proof of a steady=20\n",
      "revenue stream - a rate increase - to convince them the bonds can be repaid=\n",
      ".=20\n",
      "However, people inevitably will conserve if they are paying more for their=\n",
      "=20\n",
      "power, so the plan will also help reduce demand.=20\n",
      "Q: I'm already conserving. How can I cut my bill further?=20\n",
      "A: It's nitty-gritty time. Replace your incandescent light bulbs with=20\n",
      "fluorescent bulbs. Clean those coils at the back of your refrigerator. Turn=\n",
      "=20\n",
      "your air conditioner down or off. Reduce the amount of time you use your po=\n",
      "ol=20\n",
      "pump.=20\n",
      "Many items in your house use electricity when they are plugged in, even if=\n",
      "=20\n",
      "they're not in use. You can cut consumption by unplugging your television,=\n",
      "=20\n",
      "computer, coffeepot, toaster and blow dryer.=20\n",
      "Q. When will I pay the increase?=20\n",
      "A: The rate hike takes effect immediately. But it could take Southern=20\n",
      "California Edison a month to 45 days to implement it, so you could face=20\n",
      "retroactive charges on a follow-up bill.=20\n",
      "Q. I live in south Orange County and buy electricity from San Diego Gas &=\n",
      "=20\n",
      "Electric Co. Does this plan affect me?=20\n",
      "A: No. SDG&E has applied for a rate increase, but state regulators have not=\n",
      "=20\n",
      "yet voted on that request.=20\n",
      "Q. I'm outraged. We didn't ask for deregulation and now we're paying more.=\n",
      "=20\n",
      "How can I protest?=20\n",
      "A: Call or write you state lawmakers and the governor. You can also join on=\n",
      "e=20\n",
      "of the consumer groups fighting the rate hike, such as the Foundation for=\n",
      "=20\n",
      "Taxpayer and Consumer Rights. It is at (310) 392-0522. The Utility Reform=\n",
      "=20\n",
      "Network is at (415) 929-8876.=20\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "-----------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "National Desk; Section A=20\n",
      "RECORD RATE HIKE SET IN CALIFORNIA=20\n",
      "By EVELYN NIEVES=20\n",
      "?=20\n",
      "03/28/2001=20\n",
      "The New York Times=20\n",
      "Page 1, Column 1=20\n",
      "c. 2001 New York Times Company=20\n",
      "SAN FRANCISCO, March 27 -- California's power regulators approved the large=\n",
      "st=20\n",
      "utility rate increases in state history today in an effort to avert more=20\n",
      "rolling blackouts and keep the state's principal electricity producers from=\n",
      "=20\n",
      "collapsing in bankruptcy.=20\n",
      "Despite Gov. Gray Davis's repeated statements that no new rate increases=20\n",
      "would be needed to keep the state's two largest utilities solvent, the Stat=\n",
      "e=20\n",
      "Public Utilities Commission said today that it had no choice but to raise=\n",
      "=20\n",
      "rates by up to 46 percent for customers of the two utilities to avoid a=20\n",
      "greater catastrophe if the companies went bankrupt.=20\n",
      "''We have fought back hard in every venue possible against these unjust=20\n",
      "energy prices,'' said Loretta Lynch, the commission president and a Davis=\n",
      "=20\n",
      "appointee, who had proposed the increases on Monday.=20\n",
      "Ms. Lynch said that under a tiered rate system that would be hammered out=\n",
      "=20\n",
      "within the next 30 days, heavy power users would face the greatest increase=\n",
      "s=20\n",
      "while light users would face little or no increases. Customers will see the=\n",
      "=20\n",
      "rate increases in their May utility bills.=20\n",
      "Perhaps 45 percent of consumers under this system -- including the poorest=\n",
      "=20\n",
      "residents, who would be exempt -- will face no increase at all, Ms. Lynch=\n",
      "=20\n",
      "said. Heavy users will face increases of about 36 percent, she said.=20\n",
      "Coupled with the average 10 percent temporary rate increase in January that=\n",
      "=20\n",
      "the commission made permanent today, those consumers will pay 46 percent mo=\n",
      "re=20\n",
      "for utilities than they did in December.=20\n",
      "But even people who use small amounts of power and do not see their rates=\n",
      "=20\n",
      "increase could feel the impact if commercial power customers, from small=20\n",
      "businesses to megastores, pass on their higher electricity costs through=20\n",
      "price increases.=20\n",
      "Before the rate increases, power costs in California were slightly higher=\n",
      "=20\n",
      "than the national average, but lower than the prices paid in New York and=\n",
      "=20\n",
      "other Northeastern States. Residential electricity users already pay an=20\n",
      "average of about $65 a month, state officials say. In addition, under a=20\n",
      "previous commission decision, rates will go up an additional 10 percent nex=\n",
      "t=20\n",
      "year.=20\n",
      "The commission, which approved the rate increase 5 to 0, also voted to=20\n",
      "require that the two large utilities, the Pacific Gas and Electric Company=\n",
      "=20\n",
      "and Southern California Edison, reimburse the state's Department of Water=\n",
      "=20\n",
      "Resources $4 billion spent on their behalf and begin paying alternative=20\n",
      "energy generators, which account for more than a quarter of the state's dai=\n",
      "ly=20\n",
      "energy needs. The two utilities have not paid their bills for months, sayin=\n",
      "g=20\n",
      "they are nearly bankrupt because under deregulation of the power industry,=\n",
      "=20\n",
      "wholesalers have been allowed to float prices while retail rates were cappe=\n",
      "d.=20\n",
      "The Department of Water Resources has spent $45 million a day since January=\n",
      "=20\n",
      "buying power on the utilities' behalf. Small alternative energy producers h=\n",
      "ad=20\n",
      "not been paid at all for months, with dire consequences. For two days last=\n",
      "=20\n",
      "week, the state's power managers were forced to order rolling blackouts=20\n",
      "across parts of the state, in part because the small energy producers shut=\n",
      "=20\n",
      "down half their capacity because they had not been paid.=20\n",
      "The rate increases will make the payoff more feasible, Ms. Lynch said, whil=\n",
      "e=20\n",
      "encouraging energy conservation. But a representative of the state's=20\n",
      "alternative-energy generators, known as qualifying facilities, complained=\n",
      "=20\n",
      "that the commission did nothing to address the roughly $1.5 billion the=20\n",
      "utilities owed producers.=20\n",
      "''I'm not sure what our options are at this point if the state wants the=20\n",
      "qualified facilities to be a part of the solution,'' said David Fogarty, a=\n",
      "=20\n",
      "spokesman for the Western States Petroleum Association and alternative-ener=\n",
      "gy=20\n",
      "co-generators, representing about 50 alternative-energy producers in=20\n",
      "California.=20\n",
      "For those on all sides, the commission seemed to raise as many questions as=\n",
      "=20\n",
      "it answered. Consumer advocates, who interrupted the hearing five times,=20\n",
      "questioned why the commission had decided on the vote so abruptly, with onl=\n",
      "y=20\n",
      "24-hour notice to the public. Spokesmen for each of the utility companies=\n",
      "=20\n",
      "said they had no idea what the rate increases would mean to the average=20\n",
      "customer until the tiered system was determined. They also said they did no=\n",
      "t=20\n",
      "know whether these new rates would mean the end of rolling blackouts.=20\n",
      "''They may know something that I don't know,'' said Bruce Foster, vice=20\n",
      "president of regulatory affairs at Southern California Edison, speaking of=\n",
      "=20\n",
      "the commission's vote outside the hearing room.=20\n",
      "Mr. Foster also complained that the commission's directive to begin paying=\n",
      "=20\n",
      "bills to the state and the alternative energy producers -- or face a penalt=\n",
      "y=20\n",
      "equal to the amount owed -- might be illegal and said that the company woul=\n",
      "d=20\n",
      "look into appealing it.=20\n",
      "Ron Low, a spokesman for Pacific Gas and Electric, also complained that the=\n",
      "=20\n",
      "commission's actions seemed rushed and lacked specifics.=20\n",
      "The rate increase will generate about $2.5 billion a year for Pacific Gas a=\n",
      "nd=20\n",
      "Electric, unit of PG&E and $2.3 billion for Southern California Edison, a=\n",
      "=20\n",
      "unit of Edison International. While those revenues will help the utilities=\n",
      "=20\n",
      "buy power on the wholesale market in the future, the commission made no=20\n",
      "provision for paying off about $13 billion in debt accumulated by the=20\n",
      "utilities as a result of past wholesale costs. That issue would be left to=\n",
      "=20\n",
      "the governor and state legislators.=20\n",
      "The commission heard public comment before the vote, but did not answer=20\n",
      "questions afterward. Consumer advocates were furious, many blaming Governor=\n",
      "=20\n",
      "Davis.=20\n",
      "The governor today issued a statement calling the commission's move=20\n",
      "premature. ''From the beginning, I've said that my main goal was to solve=\n",
      "=20\n",
      "this problem while protecting the consumer from undue rate hikes,'' Mr. Dav=\n",
      "is=20\n",
      "said. ''The P.U.C.'s action today was premature because we do not have all=\n",
      "=20\n",
      "the appropriate financial numbers necessary to make a decision.''=20\n",
      "Mr. Davis's statement seemed to reflect the awareness he and his aides feel=\n",
      "=20\n",
      "about the potential political backlash from big rate increases. Even as oth=\n",
      "er=20\n",
      "politicians warned that such increases would be necessary, Mr. Davis has=20\n",
      "vowed to avoid them. On Monday he insisted that he had not spoken to any=20\n",
      "members of the commission about the rate increase proposal and that he had =\n",
      "no=20\n",
      "control over its actions.=20\n",
      "But Mr. Davis appointed three of the body's five members, and since taking=\n",
      "=20\n",
      "office two years ago he has been emphatic in saying that he expects his=20\n",
      "appointees to state boards and commissions to reflect his policies.=20\n",
      "He also did not hesitate to complain that he could not control the previous=\n",
      "=20\n",
      "members of the commission, because they were Republican appointees, and his=\n",
      "=20\n",
      "aides acknowledged that top gubernatorial advisers had consulted with Ms.=\n",
      "=20\n",
      "Lynch and approved her plans.=20\n",
      "Politicians in both parties said Mr. Davis was trying to shift blame for an=\n",
      "y=20\n",
      "rate increase to the commission while claiming that increases were approved=\n",
      "=20\n",
      "over his objections. They predicted that that effort could backfire by maki=\n",
      "ng=20\n",
      "Mr. Davis look both timorous and calculating.=20\n",
      "\n",
      "Chart: ''THE COST -- The Price of Power'' Average cost per kilowatt-hour fo=\n",
      "r=20\n",
      "residential customers for 2000. Chart shows costs for following states:=20\n",
      "Connecticut ... Maine ... Massachusetts ... Rhode Island ... Vermont ... Ne=\n",
      "w=20\n",
      "Jersey ... New York ... Pennsylvania ... Illinois ... Indiana ... Michigan=\n",
      "=20\n",
      "... Ohio ... Wisconsin ... Iowa ... Kansas ... Minnesota ... Missouri ...=\n",
      "=20\n",
      "Nebraska ... South Dakota ... Delaware ... D.C. ... Georgia ... Maryland ..=\n",
      ".=20\n",
      "North Carolina ... South Carolina ... Virginia ... West Virginia ... Alabam=\n",
      "a=20\n",
      "... Kentucky ... Mississippi ... Tennessee ... Arkansas ... Louisiana ...=\n",
      "=20\n",
      "Oklahoma ... Texas ... Arizona ... Colorado ... Idaho ... Montana ... Nevad=\n",
      "a=20\n",
      "... New Mexico ... Utah ... California ... Oregon ... Washington ... Alaska=\n",
      "=20\n",
      "... Hawaii ... Wyoming ... Florida ... New Hampshire ... North Dakota=20\n",
      "(Source: U.S. Department of Energy) (pg. A14)=20\n",
      "\n",
      "---------------------------------------------------------------------------=\n",
      "---\n",
      "----------------------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Calif. Regulators Raise Power Rates\n",
      "\n",
      "\n",
      "By KAREN GAUDETTE\n",
      "Associated Press Writer\n",
      "SAN FRANCISCO (AP) _ As protesters jeered ``Hell, no, we won't pay!''=20\n",
      "California regulators approved electricity rate increases of up to 46 perce=\n",
      "nt=20\n",
      "Tuesday to head off blackouts this summer and keep the state's two biggest=\n",
      "=20\n",
      "utilities from going under.=20\n",
      "The increases _ approved 5-0 by the Public Utilities Commission _ are the=\n",
      "=20\n",
      "biggest in California history and take effect immediately for the 25 millio=\n",
      "n=20\n",
      "people served by Pacific Gas and Electric Co. and Southern California Ediso=\n",
      "n=20\n",
      "Co.=20\n",
      "``The PUC has done all it can,'' commission president Loretta Lynch said.=\n",
      "=20\n",
      "``We have fought back hard in every venue possible against these unjust=20\n",
      "energy prices.''=20\n",
      "The PUC also ordered the utilities to pay the state for billions of dollars=\n",
      "=20\n",
      "of electricity it has bought on behalf of their customers. Just how much=20\n",
      "wasn't known; the state has not disclosed how much it has spent on various=\n",
      "=20\n",
      "long- and short-term power contracts.=20\n",
      "For the rate increase, the commission said it will create a tiered system=\n",
      "=20\n",
      "that will protect poor people and penalize customers who use the most=20\n",
      "electricity.=20\n",
      "Lynch proposed the higher rates as a way to force ``electricity hogs'' to=\n",
      "=20\n",
      "conserve and to help keep SoCal Edison and PG&E solvent.=20\n",
      "The plan is also seen as a way to protect California's budget surplus, whic=\n",
      "h=20\n",
      "is being drained of $40 million to $50 million a day as the state buys powe=\n",
      "r=20\n",
      "for the cash-strapped utilities. Since the crisis began, more than $4.2=20\n",
      "billion in taxpayer money has been spent to keep the lights on.=20\n",
      "SoCal Edison and PG&E say they have lost more than $13 billion since last=\n",
      "=20\n",
      "summer because of high wholesale electricity prices and because California'=\n",
      "s=20\n",
      "1996 deregulation law prevents the utilities from passing those costs on to=\n",
      "=20\n",
      "their customers.=20\n",
      "Both have urged the PUC to raise rates. Following the PUC meeting, SoCal=20\n",
      "Edison said it will begin paying interest to bondholders after weeks of=20\n",
      "missing payments.=20\n",
      "The rate increase will be on top of the 9 percent to 15 percent hike approv=\n",
      "ed=20\n",
      "by the PUC in January, and a 10 percent increase already scheduled for next=\n",
      "=20\n",
      "year.=20\n",
      "Ratepayers and consumer groups branded the latest plan a rip-off.=20\n",
      "``We are being held hostage by a handful of energy companies that, under=20\n",
      "deregulation, got control of our electricity supply,'' said Harvey=20\n",
      "Rosenfield, president of the Foundation for Taxpayer and Consumer Rights in=\n",
      "=20\n",
      "Santa Monica. ``Until our elected officials start acting to protect us, we=\n",
      "=20\n",
      "are going to be at their mercy, at the mercy of this ripoff.''=20\n",
      "Democratic Gov. Gray Davis has said that he opposes rate increases but that=\n",
      "=20\n",
      "he has no power over the PUC. However, three of the PUC members are his=20\n",
      "appointees. And his aides have told lawmakers that rates have to rise.=20\n",
      "Residents already pay about $65 per month for electricity _ 7.2 cents per=\n",
      "=20\n",
      "kilowatt hour to SoCal Edison and 6.5 cents per kwh to PG&E.=20\n",
      "The latest rate hike would mean a 42 percent increase for SoCal Edison=20\n",
      "customers and 46 percent for PG&E customers for electricity alone. Since th=\n",
      "e=20\n",
      "rates are bundled with other fees, the average price of a kilowatt hour is=\n",
      "=20\n",
      "closer to 12.5 cents for SoCal Edison customers and 10.5 cents for PG&E=20\n",
      "customers.=20\n",
      "Spokesmen for both utilities said it is impossible to calculate how much th=\n",
      "e=20\n",
      "plan will cost customers because the effects of the tiered system are not=\n",
      "=20\n",
      "known.=20\n",
      "The PUC meeting was disrupted at least five times by screaming protesters,=\n",
      "=20\n",
      "and dismay over the plan was seen elsewhere.=20\n",
      "Sam Sahouria, owner of the Fox Plaza Grocery and Deli in San Francisco, sai=\n",
      "d=20\n",
      "rate hikes could be too much for his family-run store to bear. ``It's=20\n",
      "terrible. We will probably go out of business,'' he said.=20\n",
      "Belinda Lazzerini, 40, who serves fruit smoothies at Jitters & Shakes in Sa=\n",
      "n=20\n",
      "Francisco, said her electric bills have jumped to $70 for her studio=20\n",
      "apartment.=20\n",
      "``The Laundromat has gone up from $1.50 to $3, so now we will have to clean=\n",
      "=20\n",
      "our clothes by hand and dry them in the basement,'' she said. ``It's crazy.=\n",
      "''=20\n",
      "The state has pledged to issue at least $10 billion in revenue bonds to buy=\n",
      "=20\n",
      "power for SoCal Edison and PG&E _ bonds that would be repaid by the=20\n",
      "utilities' customers. However, state officials have told lawmakers that sta=\n",
      "te=20\n",
      "efforts to help the utilities could hit $23 billion by 2003.=20\n",
      "California's Independent System Operator declared a Stage 2 alert Tuesday a=\n",
      "s=20\n",
      "electricity reserves dropped to about 5 percent.=20\n",
      "The alert follows two days of statewide rolling blackouts last week and two=\n",
      "=20\n",
      "others in January. ISO spokeswoman Stephanie McCorkle said many power plant=\n",
      "s=20\n",
      "that use alternative sources of energy remained down, in part because haven=\n",
      "'t=20\n",
      "been paid by the utilities.=20\n",
      "=20\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['n_words'] == 27651]['content'].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "194590dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if there are emails containing Html tags\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def hasHtml(text):\n",
    "    soup = BeautifulSoup(text, 'html.parser')\n",
    "    return bool(soup.find())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "826bcb86",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/103469 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      " 75%|████████████████████████▋        | 77343/103469 [00:06<00:01, 16041.68it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|████████████████████████████████| 103469/103469 [00:08<00:00, 12924.53it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "1       Traveling to have a business meeting takes the...      139    False  \n",
       "2                          test successful.  way to go!!!        5    False  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False  \n",
       "7                        any morning between 10 and 11:30        6    False  \n",
       "...                                                   ...      ...      ...  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87     True  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75     True  \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False  \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False  \n",
       "\n",
       "[103469 rows x 5 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check how many emails contains Html tags\n",
    "\n",
    "re_emails['hasHtml'] = re_emails['content'].progress_apply(lambda x: hasHtml(x))\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0dc87052",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    85415\n",
       "True     18054\n",
       "Name: hasHtml, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_emails['hasHtml'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e47c0eb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>&lt;27936946.1075855378542.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: 2- SURVEY - PHILLIP ALLEN</td>\n",
       "      <td>---------------------- Forwarded by Phillip K ...</td>\n",
       "      <td>223</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>&lt;15164543.1075855378954.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: This morning's Commission meeting delayed</td>\n",
       "      <td>---------------------- Forwarded by Phillip K ...</td>\n",
       "      <td>312</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>&lt;18104010.1075855725437.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: NGI access to eol</td>\n",
       "      <td>Grif,\\n\\nPlease provide a temporary id\\n\\nPhil...</td>\n",
       "      <td>93</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>&lt;28859452.1075855728121.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: SM134</td>\n",
       "      <td>---------------------- Forwarded by Phillip K ...</td>\n",
       "      <td>230</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>&lt;15982773.1075855728341.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Draft of Opposition to ORA/TURN petition</td>\n",
       "      <td>---------------------- Forwarded by Phillip K ...</td>\n",
       "      <td>160</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494511</th>\n",
       "      <td>&lt;5222805.1075842015460.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: EOL/Kiodex/energydesk.com(EDC)</td>\n",
       "      <td>fyi\\n\\n---------------------- Forwarded by Jay...</td>\n",
       "      <td>396</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494886</th>\n",
       "      <td>&lt;26657944.1075842018047.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: HoustonStreet letter</td>\n",
       "      <td>fyi\\n---------------------- Forwarded by Bob S...</td>\n",
       "      <td>328</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494910</th>\n",
       "      <td>&lt;12312929.1075842017998.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Current Status</td>\n",
       "      <td>Chris:\\n\\nAs I explained in our phone conversa...</td>\n",
       "      <td>665</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18054 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "50      <27936946.1075855378542.JavaMail.evans@thyme>   \n",
       "183     <15164543.1075855378954.JavaMail.evans@thyme>   \n",
       "359     <18104010.1075855725437.JavaMail.evans@thyme>   \n",
       "489     <28859452.1075855728121.JavaMail.evans@thyme>   \n",
       "499     <15982773.1075855728341.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "494511   <5222805.1075842015460.JavaMail.evans@thyme>   \n",
       "494886  <26657944.1075842018047.JavaMail.evans@thyme>   \n",
       "494910  <12312929.1075842017998.JavaMail.evans@thyme>   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "\n",
       "                                              subject  \\\n",
       "50                      Re: 2- SURVEY - PHILLIP ALLEN   \n",
       "183     Re: This morning's Commission meeting delayed   \n",
       "359                             Re: NGI access to eol   \n",
       "489                                         Re: SM134   \n",
       "499      Re: Draft of Opposition to ORA/TURN petition   \n",
       "...                                               ...   \n",
       "494511             Re: EOL/Kiodex/energydesk.com(EDC)   \n",
       "494886                       Re: HoustonStreet letter   \n",
       "494910                             Re: Current Status   \n",
       "495073                               Re: Dec 19 marks   \n",
       "495170                                     Re: Trades   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "50      ---------------------- Forwarded by Phillip K ...      223     True  \n",
       "183     ---------------------- Forwarded by Phillip K ...      312     True  \n",
       "359     Grif,\\n\\nPlease provide a temporary id\\n\\nPhil...       93     True  \n",
       "489     ---------------------- Forwarded by Phillip K ...      230     True  \n",
       "499     ---------------------- Forwarded by Phillip K ...      160     True  \n",
       "...                                                   ...      ...      ...  \n",
       "494511  fyi\\n\\n---------------------- Forwarded by Jay...      396     True  \n",
       "494886  fyi\\n---------------------- Forwarded by Bob S...      328     True  \n",
       "494910  Chris:\\n\\nAs I explained in our phone conversa...      665     True  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87     True  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75     True  \n",
       "\n",
       "[18054 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check sample emails containing HTML tags\n",
    "\n",
    "re_emails[re_emails['hasHtml'] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e621024c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create utility function to extract html content\n",
    "\n",
    "def extract_html_content(html):\n",
    "    soup = BeautifulSoup(html)\n",
    "    content = soup.get_text()\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2aa6944e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/103469 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      " 75%|█████████████████████████▍        | 77525/103469 [00:13<00:04, 6288.95it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|█████████████████████████████████| 103469/103469 [00:17<00:00, 5757.36it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "1       Traveling to have a business meeting takes the...      139    False  \n",
       "2                          test successful.  way to go!!!        5    False  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False  \n",
       "7                        any morning between 10 and 11:30        6    False  \n",
       "...                                                   ...      ...      ...  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87     True  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75     True  \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False  \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False  \n",
       "\n",
       "[103469 rows x 5 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract textual content from Html\n",
    "\n",
    "re_emails['content'] = re_emails['content'].progress_apply(lambda x: extract_html_content(x))\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "37f98f06",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/103469 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      " 75%|████████████████████████▋        | 77436/103469 [00:04<00:01, 19999.25it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|████████████████████████████████| 103469/103469 [00:06<00:00, 17217.65it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "1       Traveling to have a business meeting takes the...      139    False  \n",
       "2                          test successful.  way to go!!!        5    False  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False  \n",
       "7                        any morning between 10 and 11:30        6    False  \n",
       "...                                                   ...      ...      ...  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87    False  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75    False  \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False  \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False  \n",
       "\n",
       "[103469 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check again how many emails contains Html tags after extracting content\n",
    "\n",
    "re_emails['hasHtml'] = re_emails['content'].progress_apply(lambda x: hasHtml(x))\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4dfa2ee1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    103464\n",
       "True          5\n",
       "Name: hasHtml, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_emails['hasHtml'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b77ee528",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53253</th>\n",
       "      <td>&lt;8519196.1075843430568.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: FW: A.00-11-038 Rate Design Proposal of th...</td>\n",
       "      <td>Oh, that's very funny.  Just like the good old...</td>\n",
       "      <td>555</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78119</th>\n",
       "      <td>&lt;10249916.1075843879942.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: FW: A.00-11-038 Rate Design Proposal of th...</td>\n",
       "      <td>Oh, that's very funny.  Just like the good old...</td>\n",
       "      <td>555</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260726</th>\n",
       "      <td>&lt;29746531.1075852802676.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: European Expansion Plans</td>\n",
       "      <td>Original Message Follows:\\n-------------------...</td>\n",
       "      <td>773</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278637</th>\n",
       "      <td>&lt;29328335.1075861138819.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: *EMCA* here are some pics</td>\n",
       "      <td>What a neighborhood!!  The pictures were wonde...</td>\n",
       "      <td>389</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278639</th>\n",
       "      <td>&lt;13302177.1075861138866.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: *EMCA* here are some pics</td>\n",
       "      <td>I love technology!!!!!\\nNathalie\\n----- Origin...</td>\n",
       "      <td>397</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "53253    <8519196.1075843430568.JavaMail.evans@thyme>   \n",
       "78119   <10249916.1075843879942.JavaMail.evans@thyme>   \n",
       "260726  <29746531.1075852802676.JavaMail.evans@thyme>   \n",
       "278637  <29328335.1075861138819.JavaMail.evans@thyme>   \n",
       "278639  <13302177.1075861138866.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "53253   Re: FW: A.00-11-038 Rate Design Proposal of th...   \n",
       "78119   Re: FW: A.00-11-038 Rate Design Proposal of th...   \n",
       "260726                       Re: European Expansion Plans   \n",
       "278637                      Re: *EMCA* here are some pics   \n",
       "278639                      Re: *EMCA* here are some pics   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "53253   Oh, that's very funny.  Just like the good old...      555     True  \n",
       "78119   Oh, that's very funny.  Just like the good old...      555     True  \n",
       "260726  Original Message Follows:\\n-------------------...      773     True  \n",
       "278637  What a neighborhood!!  The pictures were wonde...      389     True  \n",
       "278639  I love technology!!!!!\\nNathalie\\n----- Origin...      397     True  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check remaining emails with HTML tags\n",
    "\n",
    "re_emails[re_emails['hasHtml'] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "82385143",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh, that's very funny.  Just like the good old days?\n",
      "\n",
      "\n",
      "\n",
      "\tPrentice Sellers \n",
      "\t04/16/2001 06:49 PM\n",
      "\t\t \n",
      "\t\t To: Jeff.Dasovich@enron.com\n",
      "\t\t cc: \n",
      "\t\t Subject: Re: FW: A.00-11-038 Rate Design Proposal of the Energy  Producers \n",
      "and  User s\n",
      "\n",
      "Did you want me to summarize these for you?\n",
      "\n",
      "At 08:37 PM 4/15/2001 -0500, you wrote:\n",
      "\n",
      ">----- Forwarded by Jeff Dasovich/NA/Enron on 04/15/2001 08:30 PM -----\n",
      "> \n",
      ">\n",
      ">                     JBennett \n",
      " >\n",
      ">                      (E-mail)'\"\n",
      ">                     SR.com>              , \n",
      "> \"'dblack@enron.com'\"\n",
      ">                                          , \n",
      "> \"'James.D.Steffes@enron.com'\"\n",
      ">                     04/14/2001           , \n",
      "> \"'jlewis@enron.com'\"\n",
      ">                     09:39 AM             , \n",
      "> \"'Jeff.Dasovich@enron.com'\"\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Paul.Kaufman@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Robert.Neustaedter@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Susan.J.Mara@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      "> \"'tjohnso8@enron.com'\"\n",
      ">                                          , \n",
      "> \"'Leslie.Lawner@enron.com'\"\n",
      ">                                           \n",
      " >\n",
      ">                                          cc: \n",
      " >\n",
      ">                                          Subject:     FW: A.00-11-038 \n",
      "> Rate Design Proposal\n",
      ">                                          of the Energy Producers \n",
      "> and      User s\n",
      "> \n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">-----Original Message-----\n",
      ">From: Phil Vizcarra [mailto:pav@a-klaw.com]\n",
      ">Sent: Friday, April 13, 2001 3:17 PM\n",
      ">To: jbushee@sablaw.com; kmccrea@sablaw.com; callen@akingump.com;\n",
      ">jweisgall@aol.com; rogerberliner@bcjlaw.com; rcsstl@cdmnet.com;\n",
      ">mbrubaker@consultbai.com; anchau@shellus.com;\n",
      ">keith-sappenfield@reliantenergy.com; jsmollon@newwestenergy.com;\n",
      ">jlyoung@sempra.com; napedersen@jonesday.com; athomas@newenergy.com;\n",
      ">kmcspadd@milbank.com; hchoy@isd.co.la.ca.us; lgurick@calpx.com;\n",
      ">douglass@arterhadden.com; randy_britt@mayco.com; dmarshall@greggind.com;\n",
      ">frank.cooley@sce.com; glwaas@calpx.com; uwuaregion5@earthlink.net;\n",
      ">fmo@sdcity.sannet.gov; jleslie@luce.com; julesan@aol.com;\n",
      ">mshames@ucan.org; jparrott@sempra.com; kmelville@sempra.com;\n",
      ">riald@kindermorgan.com; bill.mccallum@ci.fresno.ca.us;\n",
      ">btenney@landuselaw.com; ronknecht@aol.com;\n",
      ">furutanj@efawest.navfac.navy.mil; kpoole@adamsbroadwell.com;\n",
      ">mdjoseph@adamsbroadwell.com; freedman@turn.org; rcosta@turn.org;\n",
      ">bfinkelstein@turn.org; theresa_mueller@ci.sf.ca.us; jjz@cpuc.ca.gov;\n",
      ">chrisw@greenlining.org; joseh@lif.org; Evelyn Kahl; jnnc@chevron.com;\n",
      ">jlondon@gralegal.com; imoosen@gralegal.com; clpearce@duanemorris.com;\n",
      ">phanschen@mofo.com; tmberliner@duanemorris.com; dpritchard@mofo.com;\n",
      ">smoss@hooked.net; epoole@adplaw.com; chilen@llgm.com; jsqueri@gmssr.com;\n",
      ">jbennett@gmssr.com; bloomje@la.whitecase.com; pbray@newpower.com;\n",
      ">rick.counihan@greenmountain.com; thoulihan@mdbe.com; mday@gmssr.com;\n",
      ">jmalkin@orrick.com; edwardoneill@dwt.com; lindseyhowdowning@dwt.com;\n",
      ">jguzman@nossaman.com; whe1@pge.com; ssmyers@worldnet.att.net;\n",
      ">pxo2@pge.com; pat.williams@dhcmail.com; wbooth@booth-law.com;\n",
      ">jerryl@abag.ca.gov; mgomez1@bart.gov; askaff@energy-law-group.com;\n",
      ">pjpowerlaw@aol.com; difellman@energy-law-group.com;\n",
      ">nryan@environmentaldefense.org; rschmidt@bartlewells.com;\n",
      ">rrcollins@n-h-i.org; patrickm@crossborderenergy.com;\n",
      ">tomb@crossborderenergy.com; brbarkovich@earthlink.net;\n",
      ">pmcdonne@wenet.net; pjmuller@ricochet.net; jbradley@svmg.org;\n",
      ">tsmegal@calwater.com; sberlin@mccarthylaw.com; bishop@jrwood.com;\n",
      ">cwrmccv@worldnet.att.net; scottst@mid.org; bill@jbsenergy.com;\n",
      ">cmkehrein@ems-ca.com; jweil@aglet.org; abb@eslawfirm.com;\n",
      ">bpowers@seniors.org; dcarroll@dbsr.com; dkk@eslawfirm.com; ed@clfp.com;\n",
      ">jtachera@energy.state.ca.us; howens@seniors.org;\n",
      ">sgreenberg@realenergy.com; tknox@klalawfirm.com; smutny@iepa.com;\n",
      ">lmh@eslawfirm.com; hodgesjl@pacbell.net; rpernell@energy.state.ca.us;\n",
      ">kmills@cfbf.com; rliebert@cfbf.com; aorchar@smud.org; Michael Alcantar;\n",
      ">jcpaine@stoel.com; mhg@hetrading.com; dws@keywaycorp.com;\n",
      ">hbd@gweiss.com; gerhordt.herbert@msdw.com; cread@steptoe.com;\n",
      ">jogg@jhenergy.com; peter_fox-penner@brattle.com;\n",
      ">DavoodiKR@efaches.navfac.navy.mil; bill.wells@afcesa.af.mil;\n",
      ">ad046@detroit.freenet.org; bob_anderson@apses.com; gdreed@sidley.com;\n",
      ">robert.d.schasel@fritolay.com; cyounger@manatt.com; dhuard@manatt.com;\n",
      ">rkeen@manatt.com; georgeperrault@msn.com; picketse@sce.com;\n",
      ">jsilva@edisonenterprises.com; jherrera@dpw.co.la.ca.us;\n",
      ">LVanWagenen@sempra.com; jbarthrop@powersavers.com; foothill@lmi.net;\n",
      ">fosterbc@sce.com; ljhubbard@sempra.com; ktilton@gralegal.com;\n",
      ">mpatel@brownwoodlaw.com; paul.harris@bridge.com; oshirock@pacbell.net;\n",
      ">natutt@duanemorris.com; j3m9@pge.com; musicamaria@yahoo.com;\n",
      ">derkp@newsdata.com; luluw@newsdata.com; tom.oneill@abnamro.com;\n",
      ">mmattes@nossaman.com; RJP2@pge.com; brb3@pge.com; ell5@pge.com;\n",
      ">jyf1@pge.com; NXK2@pge.com; rrh3@pge.com; alexm@calpine.com;\n",
      ">gtbl@dynegy.com; joe.paul@dynegy.com; honest@compuserve.com;\n",
      ">jyoung@ebmud.com; jon.jacobs@paconsulting.com;\n",
      ">emitchel@angnewspapers.com; jptang@oaklandcityattorney.org;\n",
      ">rbw@mrwassoc.com; dmarcus@slip.net; jeff@hmhresources.com; ed@apx.com;\n",
      ">wendy@econinsights.com; chrism@mid.org; fwmonier@tid.org;\n",
      ">rmccann@cal.net; blaising@braunlegal.com; melanie_gillette@rmiinc.com;\n",
      ">twetzel@thermoecotek.com; Slayman@energy.state.ca.us;\n",
      ">swchurchill@juno.com; jdh@eslawfirm.com; ddavie@hesinet.com;\n",
      ">karen@klindh.com; cpeyton@sacbee.com; thaines@smud.org;\n",
      ">huse@eesconsulting.com; mer@cpuc.ca.gov; awp@cpuc.ca.gov;\n",
      ">cmw@cpuc.ca.gov; ctd@cpuc.ca.gov; bsl@cpuc.ca.gov; dlf@cpuc.ca.gov;\n",
      ">fua@cpuc.ca.gov; jlo@cpuc.ca.gov; jsw@cpuc.ca.gov; jrd@cpuc.ca.gov;\n",
      ">jmh@cpuc.ca.gov; kok@cpuc.ca.gov; kim@cpuc.ca.gov; llk@cpuc.ca.gov;\n",
      ">lss@cpuc.ca.gov; mv1@cpuc.ca.gov; pmd@cpuc.ca.gov; gig@cpuc.ca.gov;\n",
      ">rtf@cpuc.ca.gov; raw@cpuc.ca.gov; sfc@cpuc.ca.gov; car@cpuc.ca.gov;\n",
      ">txb@cpuc.ca.gov; ztc@cpuc.ca.gov; nevillm@hdcdojnet.state.ca.us;\n",
      ">dwrlegal1@water.ca.gov; ath@cpuc.ca.gov; LKristov@energy.state.ca.us;\n",
      ">randy.chinn@senate.ca.gov; firmiyas@dof.ca.gov; rax@cpuc.ca.gov;\n",
      ">rtavares@energy.state.ca.us; mschwebs@energy.state.ca.us\n",
      ">Subject: A.00-11-038 Rate Design Proposal of the Energy Producers and\n",
      ">User s\n",
      ">\n",
      ">\n",
      ">Attached pursuant to the Assigned Commissioner Ruling in A.00-11-038 et\n",
      ">al is the Rate Design Proposal of the Energy Producers and Users\n",
      ">Coalition.  The proposal includes three attachments:\n",
      ">\n",
      ">1.   Prepared Direct Testimony of Donald W. Schoenbeck on Behalf of the\n",
      ">Energy Producers and Users Coalition and Rate Spread and Rate Design\n",
      ">Proposal (Schedule 1).\n",
      ">\n",
      ">2.    Schedule 2:  EPUC ACR Template for Southern California Edison\n",
      ">Company.\n",
      ">\n",
      ">3.    Schedule 3: EPUC ACR Template for Pacific Gas and Electric\n",
      ">Company.\n",
      ">\n",
      ">Should you have any problems opening the attached documents, please let me\n",
      ">know.\n",
      ">\n",
      ">Phil Vizcarra\n",
      ">Alcantar & Kahl LLP\n",
      ">\n",
      ">  <>  <Design Testimony .doc>>  <Testimony\n",
      ">.doc>>\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">(See attached file: EPUC Rate Design Testimony .doc)\n",
      ">(See attached file: Exhibit DWS Schedule 2 EPUC Rate Design Testimony .doc)\n",
      ">(See attached file: Exhibit DWS Schedule 3 EPUC Rate Design Testimony .doc)\n",
      ">\n",
      ">\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "25595ac9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh, that's very funny.  Just like the good old days?\n",
      "\n",
      "\n",
      "\n",
      "\tPrentice Sellers \n",
      "\t04/16/2001 06:49 PM\n",
      "\t\t \n",
      "\t\t To: Jeff.Dasovich@enron.com\n",
      "\t\t cc: \n",
      "\t\t Subject: Re: FW: A.00-11-038 Rate Design Proposal of the Energy  Producers \n",
      "and  User s\n",
      "\n",
      "Did you want me to summarize these for you?\n",
      "\n",
      "At 08:37 PM 4/15/2001 -0500, you wrote:\n",
      "\n",
      ">----- Forwarded by Jeff Dasovich/NA/Enron on 04/15/2001 08:30 PM -----\n",
      "> \n",
      ">\n",
      ">                     JBennett \n",
      " >\n",
      ">                      (E-mail)'\"\n",
      ">                     SR.com>              , \n",
      "> \"'dblack@enron.com'\"\n",
      ">                                          , \n",
      "> \"'James.D.Steffes@enron.com'\"\n",
      ">                     04/14/2001           , \n",
      "> \"'jlewis@enron.com'\"\n",
      ">                     09:39 AM             , \n",
      "> \"'Jeff.Dasovich@enron.com'\"\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Paul.Kaufman@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Robert.Neustaedter@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Susan.J.Mara@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      "> \"'tjohnso8@enron.com'\"\n",
      ">                                          , \n",
      "> \"'Leslie.Lawner@enron.com'\"\n",
      ">                                           \n",
      " >\n",
      ">                                          cc: \n",
      " >\n",
      ">                                          Subject:     FW: A.00-11-038 \n",
      "> Rate Design Proposal\n",
      ">                                          of the Energy Producers \n",
      "> and      User s\n",
      "> \n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">-----Original Message-----\n",
      ">From: Phil Vizcarra [mailto:pav@a-klaw.com]\n",
      ">Sent: Friday, April 13, 2001 3:17 PM\n",
      ">To: jbushee@sablaw.com; kmccrea@sablaw.com; callen@akingump.com;\n",
      ">jweisgall@aol.com; rogerberliner@bcjlaw.com; rcsstl@cdmnet.com;\n",
      ">mbrubaker@consultbai.com; anchau@shellus.com;\n",
      ">keith-sappenfield@reliantenergy.com; jsmollon@newwestenergy.com;\n",
      ">jlyoung@sempra.com; napedersen@jonesday.com; athomas@newenergy.com;\n",
      ">kmcspadd@milbank.com; hchoy@isd.co.la.ca.us; lgurick@calpx.com;\n",
      ">douglass@arterhadden.com; randy_britt@mayco.com; dmarshall@greggind.com;\n",
      ">frank.cooley@sce.com; glwaas@calpx.com; uwuaregion5@earthlink.net;\n",
      ">fmo@sdcity.sannet.gov; jleslie@luce.com; julesan@aol.com;\n",
      ">mshames@ucan.org; jparrott@sempra.com; kmelville@sempra.com;\n",
      ">riald@kindermorgan.com; bill.mccallum@ci.fresno.ca.us;\n",
      ">btenney@landuselaw.com; ronknecht@aol.com;\n",
      ">furutanj@efawest.navfac.navy.mil; kpoole@adamsbroadwell.com;\n",
      ">mdjoseph@adamsbroadwell.com; freedman@turn.org; rcosta@turn.org;\n",
      ">bfinkelstein@turn.org; theresa_mueller@ci.sf.ca.us; jjz@cpuc.ca.gov;\n",
      ">chrisw@greenlining.org; joseh@lif.org; Evelyn Kahl; jnnc@chevron.com;\n",
      ">jlondon@gralegal.com; imoosen@gralegal.com; clpearce@duanemorris.com;\n",
      ">phanschen@mofo.com; tmberliner@duanemorris.com; dpritchard@mofo.com;\n",
      ">smoss@hooked.net; epoole@adplaw.com; chilen@llgm.com; jsqueri@gmssr.com;\n",
      ">jbennett@gmssr.com; bloomje@la.whitecase.com; pbray@newpower.com;\n",
      ">rick.counihan@greenmountain.com; thoulihan@mdbe.com; mday@gmssr.com;\n",
      ">jmalkin@orrick.com; edwardoneill@dwt.com; lindseyhowdowning@dwt.com;\n",
      ">jguzman@nossaman.com; whe1@pge.com; ssmyers@worldnet.att.net;\n",
      ">pxo2@pge.com; pat.williams@dhcmail.com; wbooth@booth-law.com;\n",
      ">jerryl@abag.ca.gov; mgomez1@bart.gov; askaff@energy-law-group.com;\n",
      ">pjpowerlaw@aol.com; difellman@energy-law-group.com;\n",
      ">nryan@environmentaldefense.org; rschmidt@bartlewells.com;\n",
      ">rrcollins@n-h-i.org; patrickm@crossborderenergy.com;\n",
      ">tomb@crossborderenergy.com; brbarkovich@earthlink.net;\n",
      ">pmcdonne@wenet.net; pjmuller@ricochet.net; jbradley@svmg.org;\n",
      ">tsmegal@calwater.com; sberlin@mccarthylaw.com; bishop@jrwood.com;\n",
      ">cwrmccv@worldnet.att.net; scottst@mid.org; bill@jbsenergy.com;\n",
      ">cmkehrein@ems-ca.com; jweil@aglet.org; abb@eslawfirm.com;\n",
      ">bpowers@seniors.org; dcarroll@dbsr.com; dkk@eslawfirm.com; ed@clfp.com;\n",
      ">jtachera@energy.state.ca.us; howens@seniors.org;\n",
      ">sgreenberg@realenergy.com; tknox@klalawfirm.com; smutny@iepa.com;\n",
      ">lmh@eslawfirm.com; hodgesjl@pacbell.net; rpernell@energy.state.ca.us;\n",
      ">kmills@cfbf.com; rliebert@cfbf.com; aorchar@smud.org; Michael Alcantar;\n",
      ">jcpaine@stoel.com; mhg@hetrading.com; dws@keywaycorp.com;\n",
      ">hbd@gweiss.com; gerhordt.herbert@msdw.com; cread@steptoe.com;\n",
      ">jogg@jhenergy.com; peter_fox-penner@brattle.com;\n",
      ">DavoodiKR@efaches.navfac.navy.mil; bill.wells@afcesa.af.mil;\n",
      ">ad046@detroit.freenet.org; bob_anderson@apses.com; gdreed@sidley.com;\n",
      ">robert.d.schasel@fritolay.com; cyounger@manatt.com; dhuard@manatt.com;\n",
      ">rkeen@manatt.com; georgeperrault@msn.com; picketse@sce.com;\n",
      ">jsilva@edisonenterprises.com; jherrera@dpw.co.la.ca.us;\n",
      ">LVanWagenen@sempra.com; jbarthrop@powersavers.com; foothill@lmi.net;\n",
      ">fosterbc@sce.com; ljhubbard@sempra.com; ktilton@gralegal.com;\n",
      ">mpatel@brownwoodlaw.com; paul.harris@bridge.com; oshirock@pacbell.net;\n",
      ">natutt@duanemorris.com; j3m9@pge.com; musicamaria@yahoo.com;\n",
      ">derkp@newsdata.com; luluw@newsdata.com; tom.oneill@abnamro.com;\n",
      ">mmattes@nossaman.com; RJP2@pge.com; brb3@pge.com; ell5@pge.com;\n",
      ">jyf1@pge.com; NXK2@pge.com; rrh3@pge.com; alexm@calpine.com;\n",
      ">gtbl@dynegy.com; joe.paul@dynegy.com; honest@compuserve.com;\n",
      ">jyoung@ebmud.com; jon.jacobs@paconsulting.com;\n",
      ">emitchel@angnewspapers.com; jptang@oaklandcityattorney.org;\n",
      ">rbw@mrwassoc.com; dmarcus@slip.net; jeff@hmhresources.com; ed@apx.com;\n",
      ">wendy@econinsights.com; chrism@mid.org; fwmonier@tid.org;\n",
      ">rmccann@cal.net; blaising@braunlegal.com; melanie_gillette@rmiinc.com;\n",
      ">twetzel@thermoecotek.com; Slayman@energy.state.ca.us;\n",
      ">swchurchill@juno.com; jdh@eslawfirm.com; ddavie@hesinet.com;\n",
      ">karen@klindh.com; cpeyton@sacbee.com; thaines@smud.org;\n",
      ">huse@eesconsulting.com; mer@cpuc.ca.gov; awp@cpuc.ca.gov;\n",
      ">cmw@cpuc.ca.gov; ctd@cpuc.ca.gov; bsl@cpuc.ca.gov; dlf@cpuc.ca.gov;\n",
      ">fua@cpuc.ca.gov; jlo@cpuc.ca.gov; jsw@cpuc.ca.gov; jrd@cpuc.ca.gov;\n",
      ">jmh@cpuc.ca.gov; kok@cpuc.ca.gov; kim@cpuc.ca.gov; llk@cpuc.ca.gov;\n",
      ">lss@cpuc.ca.gov; mv1@cpuc.ca.gov; pmd@cpuc.ca.gov; gig@cpuc.ca.gov;\n",
      ">rtf@cpuc.ca.gov; raw@cpuc.ca.gov; sfc@cpuc.ca.gov; car@cpuc.ca.gov;\n",
      ">txb@cpuc.ca.gov; ztc@cpuc.ca.gov; nevillm@hdcdojnet.state.ca.us;\n",
      ">dwrlegal1@water.ca.gov; ath@cpuc.ca.gov; LKristov@energy.state.ca.us;\n",
      ">randy.chinn@senate.ca.gov; firmiyas@dof.ca.gov; rax@cpuc.ca.gov;\n",
      ">rtavares@energy.state.ca.us; mschwebs@energy.state.ca.us\n",
      ">Subject: A.00-11-038 Rate Design Proposal of the Energy Producers and\n",
      ">User s\n",
      ">\n",
      ">\n",
      ">Attached pursuant to the Assigned Commissioner Ruling in A.00-11-038 et\n",
      ">al is the Rate Design Proposal of the Energy Producers and Users\n",
      ">Coalition.  The proposal includes three attachments:\n",
      ">\n",
      ">1.   Prepared Direct Testimony of Donald W. Schoenbeck on Behalf of the\n",
      ">Energy Producers and Users Coalition and Rate Spread and Rate Design\n",
      ">Proposal (Schedule 1).\n",
      ">\n",
      ">2.    Schedule 2:  EPUC ACR Template for Southern California Edison\n",
      ">Company.\n",
      ">\n",
      ">3.    Schedule 3: EPUC ACR Template for Pacific Gas and Electric\n",
      ">Company.\n",
      ">\n",
      ">Should you have any problems opening the attached documents, please let me\n",
      ">know.\n",
      ">\n",
      ">Phil Vizcarra\n",
      ">Alcantar & Kahl LLP\n",
      ">\n",
      ">  <>  <Design Testimony .doc>>  <Testimony\n",
      ">.doc>>\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">(See attached file: EPUC Rate Design Testimony .doc)\n",
      ">(See attached file: Exhibit DWS Schedule 2 EPUC Rate Design Testimony .doc)\n",
      ">(See attached file: Exhibit DWS Schedule 3 EPUC Rate Design Testimony .doc)\n",
      ">\n",
      ">\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c1262696",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Message Follows:\n",
      "------------------------\n",
      "\n",
      "\n",
      "European Expansion Plans\n",
      "\n",
      "<html>\n",
      "<head>\n",
      "<meta\n",
      "http-equiv=\"Content-Type\" content=\"text/html;\n",
      "charset=windows-1252\">\n",
      "<meta\n",
      "name=\"GENERATOR\" content=\"Microsoft FrontPage\n",
      "4.0\">\n",
      "<meta name=\"ProgId\"\n",
      "content=\"FrontPage.Editor.Document\">\n",
      "<title>Cumbria Inward\n",
      "Investment Agency</title>\n",
      "</head>\n",
      "<body>\n",
      "<div\n",
      "align=\"left\">\n",
      "  <table\n",
      "border=\"0\" cellpadding=\"0\" cellspacing=\"0\"\n",
      "width=\"100%\">\n",
      "   \n",
      "<tr>\n",
      "     \n",
      "<td valign=\"top\"\n",
      "align=\"left\">To:<br>\n",
      "        Mr. Kenneth L.\n",
      "Lay  <br>\n",
      "        COB/CEO \n",
      "<br>\n",
      "        Enron\n",
      "Corp.  <br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "From:<br>\n",
      "        Leighton\n",
      "Wildrick<br>\n",
      "        President -\n",
      "North America<br>\n",
      "        Cumbria Inward\n",
      "Investment Agency<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "        Dear Mr.\n",
      "Lay,<br>\n",
      "       \n",
      "<br>\n",
      "        A colleague of\n",
      "mine in England indicated that Enron Corp. was considering expanding in\n",
      "Europe and that England was in the mix of location options. In the\n",
      "course of your investment deliberations it is worth considering Cumbria\n",
      "in northwest England. Located two hours north of Manchester in the\n",
      "magnificent Lake District of the United Kingdom, Cumbria offers American\n",
      "companies a prime location for manufacturing, distribution and customer\n",
      "service faculties. <br>\n",
      "       \n",
      "<br>\n",
      "        Cumbria County\n",
      "is home to dozens of U.S. firms who chose to locate in the County for a\n",
      "host of reasons including a stable, educated, motivated workforce, very\n",
      "competitive real estate prices and operating costs that are among the\n",
      "lowest in Europe. Located away from congested and expensive urban\n",
      "centers; within a couple of hours of international airports in\n",
      "Manchester, Edinburgh and Glasgow, Cumbria combines low operating costs\n",
      "and a secure \"out of town\" feel with an excellent\n",
      "logistical infrastructure assuring\n",
      "cost effective access to markets all over Europe.\n",
      "        <a\n",
      "href=\"http://www.cum\n",
      "bria-investment.com/video/index.html\"\n",
      "target=\"_blank\"> Click here</a> to view an online\n",
      "video of Cumbria for a first hand look at the\n",
      "County.<br>\n",
      "       \n",
      "<br>\n",
      "        Cumbria Inward\n",
      "Investment Agency is the County-wide organization tasked with providing\n",
      "free and confidential assistance to firms considering Cumbria as a\n",
      "location for their business. CIIA has an office near Philadelphia,\n",
      "Pennsylvania staffed by inward investment professionals who have worked\n",
      "with scores of American companies expanding to Europe over the last 15\n",
      "years. The U.S. office is responsible for coordinating the exchange of\n",
      "information between your company and the professionals \"on\n",
      "the ground\" in Cumbria. Please\n",
      "give us a call or drop us an email if we can be of help as you go\n",
      "through the process of finding the best European location for your\n",
      "business. In the meantime please visit our web site at\n",
      "        <a\n",
      "href=\"http://www.cumbria-investment.c\n",
      "om\"\n",
      "target=\"_blank\">http://www.cumbria-investment.c\n",
      "om</a>.<br>\n",
      "       \n",
      "<br>\n",
      "        I look forward\n",
      "to hearing from you.<br>\n",
      "       \n",
      "<br>\n",
      "        Sincerely\n",
      "yours,<br>\n",
      "       \n",
      "<br>\n",
      "        Leighton\n",
      "Wildrick<br>\n",
      "        President -\n",
      "North America<br>\n",
      "        Cumbria Inward\n",
      "Investment Agency<br>\n",
      "        Email:\n",
      "leightonw@cumbria-investment.com<br>\n",
      "        Phone:\n",
      "215-836-0300<br>\n",
      "        Toll Free:\n",
      "1-800-600-4008<br>\n",
      "        Fax:\n",
      "215-836-9988<br>\n",
      "        http://www.cumbria-investment.c\n",
      "om<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "        Further\n",
      "transmission of unsolicited email to this recipient by the sender can be\n",
      "terminated by clicking <a href=\"http://www.cumbria-\n",
      "\n",
      "investment.com/cgi-bin/postoffice/manager.cgi?action=delete&email=info%4\n",
      "0enron.com&group1=CIIAEml3\">here</a>.</p><\n",
      "/td>\n",
      "     \n",
      "<td width=\"60\" valign=\"top\"\n",
      "align=\"right\"><img border=\"0\" src=\"http://www\n",
      ".cumbria-investment.com/images/usaribbon.gif\"\n",
      "width=\"56\" height=\"100\"></td>\n",
      "   \n",
      "</tr>\n",
      "  </table>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a7fb1305",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What a neighborhood!!  The pictures were wonderful.  I knew I wouldn't have any trouble printing the .jpg files, but I was surprised how easy it was to print from the .pdf file.  Each picture was a page.  I'm explaining that this is a test case: we want to see if District D can solve problems for us.  Ada Edwards has a great idea to set up a little neighborhood office next to the storefront police station.  I'm willing to help, but only if we can get results.  We don't need a public relations gimmick.\n",
      "\n",
      "\n",
      "Hilary\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "At 03:03 PM 3/1/2002 -0800, you wrote:\n",
      "\n",
      ">i took 3 pics\n",
      "\n",
      ">michel\n",
      "\n",
      ">--- Hilary Smith < wrote:\n",
      "\n",
      ">> I just took some pictures of Lake Bomar using old\n",
      "\n",
      ">> fashioned film which we\n",
      "\n",
      ">> probably won't develop for weeks.  If someone has a\n",
      "\n",
      ">> digital camera and\n",
      "\n",
      ">> could e-mail me a couple of pictures, I could print\n",
      "\n",
      ">> them and take them to\n",
      "\n",
      ">> Ada Edwards' office Monday.\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Hilary\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> At 02:35 PM 3/1/2002 -0600, you wrote: \n",
      "\n",
      ">> \n",
      "\n",
      ">> >>>>\n",
      "\n",
      ">> \n",
      "\n",
      ">> <anyone interested in seeing how swampy the\n",
      "\n",
      ">> lake gets - drive\n",
      "\n",
      ">> down Bomar toward taft and you'll see the \"site\" of\n",
      "\n",
      ">> discussion. A little\n",
      "\n",
      ">> rain goes a long way...\n",
      "\n",
      ">> \n",
      "\n",
      ">> Nathalie\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<0000,3333,9999<<Yahoo!\n",
      "\n",
      ">> Groups\n",
      "\n",
      ">> Sponsor<<<  \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<arial<<ADVERTISEMENT\n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> <\n",
      "\n",
      ">> \n",
      "\n",
      ">> To unsubscribe from this group, send an email to:\n",
      "\n",
      ">> \n",
      "\n",
      ">> emca-unsubscribe@egroups.com\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Your use of Yahoo! Groups is subject to the\n",
      "\n",
      ">> <<<Yahoo! Terms of\n",
      "\n",
      ">> Service.   \n",
      "\n",
      ">> \n",
      "\n",
      ">> <<<<<<<<<<<<<<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> (Mr.) Hilary Smith\n",
      "\n",
      ">> \n",
      "\n",
      ">> 712 Bomar St.\n",
      "\n",
      ">> \n",
      "\n",
      ">> Houston TX  77006-1316\n",
      "\n",
      ">> \n",
      "\n",
      ">> 713.523.0559\n",
      "\n",
      ">> \n",
      "\n",
      ">> Fax: 713.523.3928\n",
      "\n",
      ">> \n",
      "\n",
      ">> Website:\n",
      "\n",
      ">>\n",
      "\n",
      "><<<0000,0000,ffff<http://pws.prserv.net/hilary<<\n",
      "\n",
      ">> \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">__________________________________________________\n",
      "\n",
      ">Do You Yahoo!?\n",
      "\n",
      ">Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "\n",
      ">http://greetings.yahoo.com\n",
      "\n",
      ">------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "\n",
      ">Buy Stock for $4.\n",
      "\n",
      ">No Minimums.\n",
      "\n",
      ">FREE Money 2002.\n",
      "\n",
      ">http://us.click.yahoo.com/BgmYkB/VovDAA/ySSFAA/NoOolB/TM\n",
      "\n",
      ">---------------------------------------------------------------------~->\n",
      "\n",
      ">\n",
      "\n",
      ">To unsubscribe from this group, send an email to:\n",
      "\n",
      ">emca-unsubscribe@egroups.com\n",
      "\n",
      ">\n",
      "\n",
      "> \n",
      "\n",
      ">\n",
      "\n",
      ">Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/ \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood1.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood2.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood3.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      "(Mr.) Hilary Smith\n",
      "\n",
      "712 Bomar St.\n",
      "\n",
      "Houston TX  77006-1316\n",
      "\n",
      "713.523.0559\n",
      "\n",
      "Fax: 713.523.3928\n",
      "\n",
      "Website:\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f3a7316a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I love technology!!!!!\n",
      "Nathalie\n",
      "----- Original Message -----\n",
      "From: \"Michel Cahier\" \n",
      "To: \n",
      "Sent: Friday, March 01, 2002 5:03 PM\n",
      "Subject: *EMCA* here are some pics\n",
      "\n",
      "\n",
      "> i took 3 pics\n",
      "> michel\n",
      "> --- Hilary Smith  wrote:\n",
      "> > I just took some pictures of Lake Bomar using old\n",
      "> > fashioned film which we\n",
      "> > probably won't develop for weeks.  If someone has a\n",
      "> > digital camera and\n",
      "> > could e-mail me a couple of pictures, I could print\n",
      "> > them and take them to\n",
      "> > Ada Edwards' office Monday.\n",
      "> >\n",
      "> >\n",
      "> > Hilary\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> > At 02:35 PM 3/1/2002 -0600, you wrote:\n",
      "> >\n",
      "> > >>>>\n",
      "> >\n",
      "> > anyone interested in seeing how swampy the\n",
      "> > lake gets - drive\n",
      "> > down Bomar toward taft and you'll see the \"site\" of\n",
      "> > discussion. A little\n",
      "> > rain goes a long way...\n",
      "> >\n",
      "> > Nathalie\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> 0000,3333,9999Yahoo!\n",
      "> > Groups\n",
      "> > Sponsor\n",
      "> >\n",
      "> >\n",
      "> arialADVERTISEMENT\n",
      "> >\n",
      "> >\n",
      ">\n",
      "<\n",
      "> >\n",
      "> > \n",
      "> >\n",
      "> > To unsubscribe from this group, send an email to:\n",
      "> >\n",
      "> > emca-unsubscribe@egroups.com\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> > Your use of Yahoo! Groups is subject to the\n",
      "> > <Yahoo! Terms of\n",
      "> > Service.\n",
      "> >\n",
      "> > <<<<<<<<\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> > (Mr.) Hilary Smith\n",
      "> >\n",
      "> > 712 Bomar St.\n",
      "> >\n",
      "> > Houston TX  77006-1316\n",
      "> >\n",
      "> > 713.523.0559\n",
      "> >\n",
      "> > Fax: 713.523.3928\n",
      "> >\n",
      "> > Website:\n",
      "> >\n",
      ">\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary<\n",
      "/color>\n",
      "> >\n",
      ">\n",
      ">\n",
      "> __________________________________________________\n",
      "> Do You Yahoo!?\n",
      "> Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "> http://greetings.yahoo.com\n",
      ">\n",
      "> To unsubscribe from this group, send an email to:\n",
      "> emca-unsubscribe@egroups.com\n",
      ">\n",
      ">\n",
      ">\n",
      "> Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n",
      ">\n",
      ">\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "FREE COLLEGE MONEY\n",
      "CLICK HERE to search\n",
      "600,000 scholarships!\n",
      "http://us.click.yahoo.com/iZp8OC/4m7CAA/ySSFAA/NoOolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "emca-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ea045f41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can remove these when we extract the original message\n",
    "\n",
    "def extract_content(body, stop_regexes):\n",
    "    \"\"\"\n",
    "    Apply regex rules to extract\n",
    "    content\n",
    "    \"\"\"\n",
    "    content = \"\"\n",
    "    \n",
    "    #Split the body of the message into lines\n",
    "    for line in body.splitlines():\n",
    "        #If we find one of the stop expressions, \n",
    "        # stop processing and return immediately\n",
    "        for stop_regex in stop_regexes:\n",
    "            if(stop_regex.search(line)) != None:\n",
    "                return content\n",
    "        \n",
    "        content += line\n",
    "        \n",
    "        #add the line breaks again\n",
    "        content += '\\n'\n",
    "    return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cd19950c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh, that's very funny.  Just like the good old days?\n",
      "\n",
      "\n",
      "\n",
      "\tPrentice Sellers \n",
      "\t04/16/2001 06:49 PM\n",
      "\t\t \n",
      "\t\t To: Jeff.Dasovich@enron.com\n",
      "\t\t cc: \n",
      "\t\t Subject: Re: FW: A.00-11-038 Rate Design Proposal of the Energy  Producers \n",
      "and  User s\n",
      "\n",
      "Did you want me to summarize these for you?\n",
      "\n",
      "At 08:37 PM 4/15/2001 -0500, you wrote:\n",
      "\n",
      ">----- Forwarded by Jeff Dasovich/NA/Enron on 04/15/2001 08:30 PM -----\n",
      "> \n",
      ">\n",
      ">                     JBennett \n",
      " >\n",
      ">                      (E-mail)'\"\n",
      ">                     SR.com>              , \n",
      "> \"'dblack@enron.com'\"\n",
      ">                                          , \n",
      "> \"'James.D.Steffes@enron.com'\"\n",
      ">                     04/14/2001           , \n",
      "> \"'jlewis@enron.com'\"\n",
      ">                     09:39 AM             , \n",
      "> \"'Jeff.Dasovich@enron.com'\"\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Paul.Kaufman@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Robert.Neustaedter@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      " >\n",
      ">                                          \"'Susan.J.Mara@enron.com'\" \n",
      " >\n",
      ">                                          , \n",
      "> \"'tjohnso8@enron.com'\"\n",
      ">                                          , \n",
      "> \"'Leslie.Lawner@enron.com'\"\n",
      ">                                           \n",
      " >\n",
      ">                                          cc: \n",
      " >\n",
      ">                                          Subject:     FW: A.00-11-038 \n",
      "> Rate Design Proposal\n",
      ">                                          of the Energy Producers \n",
      "> and      User s\n",
      "> \n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">-----Original Message-----\n",
      ">From: Phil Vizcarra [mailto:pav@a-klaw.com]\n",
      ">Sent: Friday, April 13, 2001 3:17 PM\n",
      ">To: jbushee@sablaw.com; kmccrea@sablaw.com; callen@akingump.com;\n",
      ">jweisgall@aol.com; rogerberliner@bcjlaw.com; rcsstl@cdmnet.com;\n",
      ">mbrubaker@consultbai.com; anchau@shellus.com;\n",
      ">keith-sappenfield@reliantenergy.com; jsmollon@newwestenergy.com;\n",
      ">jlyoung@sempra.com; napedersen@jonesday.com; athomas@newenergy.com;\n",
      ">kmcspadd@milbank.com; hchoy@isd.co.la.ca.us; lgurick@calpx.com;\n",
      ">douglass@arterhadden.com; randy_britt@mayco.com; dmarshall@greggind.com;\n",
      ">frank.cooley@sce.com; glwaas@calpx.com; uwuaregion5@earthlink.net;\n",
      ">fmo@sdcity.sannet.gov; jleslie@luce.com; julesan@aol.com;\n",
      ">mshames@ucan.org; jparrott@sempra.com; kmelville@sempra.com;\n",
      ">riald@kindermorgan.com; bill.mccallum@ci.fresno.ca.us;\n",
      ">btenney@landuselaw.com; ronknecht@aol.com;\n",
      ">furutanj@efawest.navfac.navy.mil; kpoole@adamsbroadwell.com;\n",
      ">mdjoseph@adamsbroadwell.com; freedman@turn.org; rcosta@turn.org;\n",
      ">bfinkelstein@turn.org; theresa_mueller@ci.sf.ca.us; jjz@cpuc.ca.gov;\n",
      ">chrisw@greenlining.org; joseh@lif.org; Evelyn Kahl; jnnc@chevron.com;\n",
      ">jlondon@gralegal.com; imoosen@gralegal.com; clpearce@duanemorris.com;\n",
      ">phanschen@mofo.com; tmberliner@duanemorris.com; dpritchard@mofo.com;\n",
      ">smoss@hooked.net; epoole@adplaw.com; chilen@llgm.com; jsqueri@gmssr.com;\n",
      ">jbennett@gmssr.com; bloomje@la.whitecase.com; pbray@newpower.com;\n",
      ">rick.counihan@greenmountain.com; thoulihan@mdbe.com; mday@gmssr.com;\n",
      ">jmalkin@orrick.com; edwardoneill@dwt.com; lindseyhowdowning@dwt.com;\n",
      ">jguzman@nossaman.com; whe1@pge.com; ssmyers@worldnet.att.net;\n",
      ">pxo2@pge.com; pat.williams@dhcmail.com; wbooth@booth-law.com;\n",
      ">jerryl@abag.ca.gov; mgomez1@bart.gov; askaff@energy-law-group.com;\n",
      ">pjpowerlaw@aol.com; difellman@energy-law-group.com;\n",
      ">nryan@environmentaldefense.org; rschmidt@bartlewells.com;\n",
      ">rrcollins@n-h-i.org; patrickm@crossborderenergy.com;\n",
      ">tomb@crossborderenergy.com; brbarkovich@earthlink.net;\n",
      ">pmcdonne@wenet.net; pjmuller@ricochet.net; jbradley@svmg.org;\n",
      ">tsmegal@calwater.com; sberlin@mccarthylaw.com; bishop@jrwood.com;\n",
      ">cwrmccv@worldnet.att.net; scottst@mid.org; bill@jbsenergy.com;\n",
      ">cmkehrein@ems-ca.com; jweil@aglet.org; abb@eslawfirm.com;\n",
      ">bpowers@seniors.org; dcarroll@dbsr.com; dkk@eslawfirm.com; ed@clfp.com;\n",
      ">jtachera@energy.state.ca.us; howens@seniors.org;\n",
      ">sgreenberg@realenergy.com; tknox@klalawfirm.com; smutny@iepa.com;\n",
      ">lmh@eslawfirm.com; hodgesjl@pacbell.net; rpernell@energy.state.ca.us;\n",
      ">kmills@cfbf.com; rliebert@cfbf.com; aorchar@smud.org; Michael Alcantar;\n",
      ">jcpaine@stoel.com; mhg@hetrading.com; dws@keywaycorp.com;\n",
      ">hbd@gweiss.com; gerhordt.herbert@msdw.com; cread@steptoe.com;\n",
      ">jogg@jhenergy.com; peter_fox-penner@brattle.com;\n",
      ">DavoodiKR@efaches.navfac.navy.mil; bill.wells@afcesa.af.mil;\n",
      ">ad046@detroit.freenet.org; bob_anderson@apses.com; gdreed@sidley.com;\n",
      ">robert.d.schasel@fritolay.com; cyounger@manatt.com; dhuard@manatt.com;\n",
      ">rkeen@manatt.com; georgeperrault@msn.com; picketse@sce.com;\n",
      ">jsilva@edisonenterprises.com; jherrera@dpw.co.la.ca.us;\n",
      ">LVanWagenen@sempra.com; jbarthrop@powersavers.com; foothill@lmi.net;\n",
      ">fosterbc@sce.com; ljhubbard@sempra.com; ktilton@gralegal.com;\n",
      ">mpatel@brownwoodlaw.com; paul.harris@bridge.com; oshirock@pacbell.net;\n",
      ">natutt@duanemorris.com; j3m9@pge.com; musicamaria@yahoo.com;\n",
      ">derkp@newsdata.com; luluw@newsdata.com; tom.oneill@abnamro.com;\n",
      ">mmattes@nossaman.com; RJP2@pge.com; brb3@pge.com; ell5@pge.com;\n",
      ">jyf1@pge.com; NXK2@pge.com; rrh3@pge.com; alexm@calpine.com;\n",
      ">gtbl@dynegy.com; joe.paul@dynegy.com; honest@compuserve.com;\n",
      ">jyoung@ebmud.com; jon.jacobs@paconsulting.com;\n",
      ">emitchel@angnewspapers.com; jptang@oaklandcityattorney.org;\n",
      ">rbw@mrwassoc.com; dmarcus@slip.net; jeff@hmhresources.com; ed@apx.com;\n",
      ">wendy@econinsights.com; chrism@mid.org; fwmonier@tid.org;\n",
      ">rmccann@cal.net; blaising@braunlegal.com; melanie_gillette@rmiinc.com;\n",
      ">twetzel@thermoecotek.com; Slayman@energy.state.ca.us;\n",
      ">swchurchill@juno.com; jdh@eslawfirm.com; ddavie@hesinet.com;\n",
      ">karen@klindh.com; cpeyton@sacbee.com; thaines@smud.org;\n",
      ">huse@eesconsulting.com; mer@cpuc.ca.gov; awp@cpuc.ca.gov;\n",
      ">cmw@cpuc.ca.gov; ctd@cpuc.ca.gov; bsl@cpuc.ca.gov; dlf@cpuc.ca.gov;\n",
      ">fua@cpuc.ca.gov; jlo@cpuc.ca.gov; jsw@cpuc.ca.gov; jrd@cpuc.ca.gov;\n",
      ">jmh@cpuc.ca.gov; kok@cpuc.ca.gov; kim@cpuc.ca.gov; llk@cpuc.ca.gov;\n",
      ">lss@cpuc.ca.gov; mv1@cpuc.ca.gov; pmd@cpuc.ca.gov; gig@cpuc.ca.gov;\n",
      ">rtf@cpuc.ca.gov; raw@cpuc.ca.gov; sfc@cpuc.ca.gov; car@cpuc.ca.gov;\n",
      ">txb@cpuc.ca.gov; ztc@cpuc.ca.gov; nevillm@hdcdojnet.state.ca.us;\n",
      ">dwrlegal1@water.ca.gov; ath@cpuc.ca.gov; LKristov@energy.state.ca.us;\n",
      ">randy.chinn@senate.ca.gov; firmiyas@dof.ca.gov; rax@cpuc.ca.gov;\n",
      ">rtavares@energy.state.ca.us; mschwebs@energy.state.ca.us\n",
      ">Subject: A.00-11-038 Rate Design Proposal of the Energy Producers and\n",
      ">User s\n",
      ">\n",
      ">\n",
      ">Attached pursuant to the Assigned Commissioner Ruling in A.00-11-038 et\n",
      ">al is the Rate Design Proposal of the Energy Producers and Users\n",
      ">Coalition.  The proposal includes three attachments:\n",
      ">\n",
      ">1.   Prepared Direct Testimony of Donald W. Schoenbeck on Behalf of the\n",
      ">Energy Producers and Users Coalition and Rate Spread and Rate Design\n",
      ">Proposal (Schedule 1).\n",
      ">\n",
      ">2.    Schedule 2:  EPUC ACR Template for Southern California Edison\n",
      ">Company.\n",
      ">\n",
      ">3.    Schedule 3: EPUC ACR Template for Pacific Gas and Electric\n",
      ">Company.\n",
      ">\n",
      ">Should you have any problems opening the attached documents, please let me\n",
      ">know.\n",
      ">\n",
      ">Phil Vizcarra\n",
      ">Alcantar & Kahl LLP\n",
      ">\n",
      ">  <>  <Design Testimony .doc>>  <Testimony\n",
      ">.doc>>\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">\n",
      ">(See attached file: EPUC Rate Design Testimony .doc)\n",
      ">(See attached file: Exhibit DWS Schedule 2 EPUC Rate Design Testimony .doc)\n",
      ">(See attached file: Exhibit DWS Schedule 3 EPUC Rate Design Testimony .doc)\n",
      ">\n",
      ">\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "38afeada",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh, that's very funny.  Just like the good old days?\n",
      "\n",
      "\n",
      "\n",
      "\tPrentice Sellers \n",
      "\t04/16/2001 06:49 PM\n",
      "\t\t \n",
      "\t\t To: Jeff.Dasovich@enron.com\n",
      "\t\t cc: \n",
      "\t\t Subject: Re: FW: A.00-11-038 Rate Design Proposal of the Energy  Producers \n",
      "and  User s\n",
      "\n",
      "Did you want me to summarize these for you?\n",
      "\n",
      "At 08:37 PM 4/15/2001 -0500, you wrote:\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(extract_content(\n",
    "    re_emails[re_emails['hasHtml'] == True]['content'].iloc[0], \n",
    "    stop_regexes=[\n",
    "        re.compile('\\s*Original Message'),\n",
    "        re.compile('\\s*Forwarded by')\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2a7c8ee2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh, that's very funny.  Just like the good old days?\n",
      "\n",
      "\n",
      "\n",
      "\tPrentice Sellers \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(extract_content(\n",
    "    re_emails[re_emails['hasHtml'] == True]['content'].iloc[0], \n",
    "    stop_regexes=[\n",
    "        re.compile('\\s*Original Message'),\n",
    "        re.compile('\\s*Forwarded by'),\n",
    "        re.compile('^\\s?>?(From|To):\\s?', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2} (AM|PM)', re.I),\n",
    "        re.compile('\\son \\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}:\\d{2} (AM|PM)$', re.I)\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e2b29f59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Message Follows:\n",
      "------------------------\n",
      "\n",
      "\n",
      "European Expansion Plans\n",
      "\n",
      "<html>\n",
      "<head>\n",
      "<meta\n",
      "http-equiv=\"Content-Type\" content=\"text/html;\n",
      "charset=windows-1252\">\n",
      "<meta\n",
      "name=\"GENERATOR\" content=\"Microsoft FrontPage\n",
      "4.0\">\n",
      "<meta name=\"ProgId\"\n",
      "content=\"FrontPage.Editor.Document\">\n",
      "<title>Cumbria Inward\n",
      "Investment Agency</title>\n",
      "</head>\n",
      "<body>\n",
      "<div\n",
      "align=\"left\">\n",
      "  <table\n",
      "border=\"0\" cellpadding=\"0\" cellspacing=\"0\"\n",
      "width=\"100%\">\n",
      "   \n",
      "<tr>\n",
      "     \n",
      "<td valign=\"top\"\n",
      "align=\"left\">To:<br>\n",
      "        Mr. Kenneth L.\n",
      "Lay  <br>\n",
      "        COB/CEO \n",
      "<br>\n",
      "        Enron\n",
      "Corp.  <br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "From:<br>\n",
      "        Leighton\n",
      "Wildrick<br>\n",
      "        President -\n",
      "North America<br>\n",
      "        Cumbria Inward\n",
      "Investment Agency<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "        Dear Mr.\n",
      "Lay,<br>\n",
      "       \n",
      "<br>\n",
      "        A colleague of\n",
      "mine in England indicated that Enron Corp. was considering expanding in\n",
      "Europe and that England was in the mix of location options. In the\n",
      "course of your investment deliberations it is worth considering Cumbria\n",
      "in northwest England. Located two hours north of Manchester in the\n",
      "magnificent Lake District of the United Kingdom, Cumbria offers American\n",
      "companies a prime location for manufacturing, distribution and customer\n",
      "service faculties. <br>\n",
      "       \n",
      "<br>\n",
      "        Cumbria County\n",
      "is home to dozens of U.S. firms who chose to locate in the County for a\n",
      "host of reasons including a stable, educated, motivated workforce, very\n",
      "competitive real estate prices and operating costs that are among the\n",
      "lowest in Europe. Located away from congested and expensive urban\n",
      "centers; within a couple of hours of international airports in\n",
      "Manchester, Edinburgh and Glasgow, Cumbria combines low operating costs\n",
      "and a secure \"out of town\" feel with an excellent\n",
      "logistical infrastructure assuring\n",
      "cost effective access to markets all over Europe.\n",
      "        <a\n",
      "href=\"http://www.cum\n",
      "bria-investment.com/video/index.html\"\n",
      "target=\"_blank\"> Click here</a> to view an online\n",
      "video of Cumbria for a first hand look at the\n",
      "County.<br>\n",
      "       \n",
      "<br>\n",
      "        Cumbria Inward\n",
      "Investment Agency is the County-wide organization tasked with providing\n",
      "free and confidential assistance to firms considering Cumbria as a\n",
      "location for their business. CIIA has an office near Philadelphia,\n",
      "Pennsylvania staffed by inward investment professionals who have worked\n",
      "with scores of American companies expanding to Europe over the last 15\n",
      "years. The U.S. office is responsible for coordinating the exchange of\n",
      "information between your company and the professionals \"on\n",
      "the ground\" in Cumbria. Please\n",
      "give us a call or drop us an email if we can be of help as you go\n",
      "through the process of finding the best European location for your\n",
      "business. In the meantime please visit our web site at\n",
      "        <a\n",
      "href=\"http://www.cumbria-investment.c\n",
      "om\"\n",
      "target=\"_blank\">http://www.cumbria-investment.c\n",
      "om</a>.<br>\n",
      "       \n",
      "<br>\n",
      "        I look forward\n",
      "to hearing from you.<br>\n",
      "       \n",
      "<br>\n",
      "        Sincerely\n",
      "yours,<br>\n",
      "       \n",
      "<br>\n",
      "        Leighton\n",
      "Wildrick<br>\n",
      "        President -\n",
      "North America<br>\n",
      "        Cumbria Inward\n",
      "Investment Agency<br>\n",
      "        Email:\n",
      "leightonw@cumbria-investment.com<br>\n",
      "        Phone:\n",
      "215-836-0300<br>\n",
      "        Toll Free:\n",
      "1-800-600-4008<br>\n",
      "        Fax:\n",
      "215-836-9988<br>\n",
      "        http://www.cumbria-investment.c\n",
      "om<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "       \n",
      "<br>\n",
      "        Further\n",
      "transmission of unsolicited email to this recipient by the sender can be\n",
      "terminated by clicking <a href=\"http://www.cumbria-\n",
      "\n",
      "investment.com/cgi-bin/postoffice/manager.cgi?action=delete&email=info%4\n",
      "0enron.com&group1=CIIAEml3\">here</a>.</p><\n",
      "/td>\n",
      "     \n",
      "<td width=\"60\" valign=\"top\"\n",
      "align=\"right\"><img border=\"0\" src=\"http://www\n",
      ".cumbria-investment.com/images/usaribbon.gif\"\n",
      "width=\"56\" height=\"100\"></td>\n",
      "   \n",
      "</tr>\n",
      "  </table>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      "\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "dc034bff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(extract_content(\n",
    "    re_emails[re_emails['hasHtml'] == True]['content'].iloc[2], \n",
    "    stop_regexes=[\n",
    "        re.compile('\\s*Original Message'),\n",
    "        re.compile('\\s*Forwarded by'),\n",
    "        re.compile('^\\s?>?(From|To):\\s?', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2} (AM|PM)', re.I),\n",
    "        re.compile('\\son \\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}:\\d{2} (AM|PM)$', re.I)\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "dc1e35d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What a neighborhood!!  The pictures were wonderful.  I knew I wouldn't have any trouble printing the .jpg files, but I was surprised how easy it was to print from the .pdf file.  Each picture was a page.  I'm explaining that this is a test case: we want to see if District D can solve problems for us.  Ada Edwards has a great idea to set up a little neighborhood office next to the storefront police station.  I'm willing to help, but only if we can get results.  We don't need a public relations gimmick.\n",
      "\n",
      "\n",
      "Hilary\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "At 03:03 PM 3/1/2002 -0800, you wrote:\n",
      "\n",
      ">i took 3 pics\n",
      "\n",
      ">michel\n",
      "\n",
      ">--- Hilary Smith < wrote:\n",
      "\n",
      ">> I just took some pictures of Lake Bomar using old\n",
      "\n",
      ">> fashioned film which we\n",
      "\n",
      ">> probably won't develop for weeks.  If someone has a\n",
      "\n",
      ">> digital camera and\n",
      "\n",
      ">> could e-mail me a couple of pictures, I could print\n",
      "\n",
      ">> them and take them to\n",
      "\n",
      ">> Ada Edwards' office Monday.\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Hilary\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> At 02:35 PM 3/1/2002 -0600, you wrote: \n",
      "\n",
      ">> \n",
      "\n",
      ">> >>>>\n",
      "\n",
      ">> \n",
      "\n",
      ">> <anyone interested in seeing how swampy the\n",
      "\n",
      ">> lake gets - drive\n",
      "\n",
      ">> down Bomar toward taft and you'll see the \"site\" of\n",
      "\n",
      ">> discussion. A little\n",
      "\n",
      ">> rain goes a long way...\n",
      "\n",
      ">> \n",
      "\n",
      ">> Nathalie\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<0000,3333,9999<<Yahoo!\n",
      "\n",
      ">> Groups\n",
      "\n",
      ">> Sponsor<<<  \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<arial<<ADVERTISEMENT\n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> <\n",
      "\n",
      ">> \n",
      "\n",
      ">> To unsubscribe from this group, send an email to:\n",
      "\n",
      ">> \n",
      "\n",
      ">> emca-unsubscribe@egroups.com\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Your use of Yahoo! Groups is subject to the\n",
      "\n",
      ">> <<<Yahoo! Terms of\n",
      "\n",
      ">> Service.   \n",
      "\n",
      ">> \n",
      "\n",
      ">> <<<<<<<<<<<<<<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> (Mr.) Hilary Smith\n",
      "\n",
      ">> \n",
      "\n",
      ">> 712 Bomar St.\n",
      "\n",
      ">> \n",
      "\n",
      ">> Houston TX  77006-1316\n",
      "\n",
      ">> \n",
      "\n",
      ">> 713.523.0559\n",
      "\n",
      ">> \n",
      "\n",
      ">> Fax: 713.523.3928\n",
      "\n",
      ">> \n",
      "\n",
      ">> Website:\n",
      "\n",
      ">>\n",
      "\n",
      "><<<0000,0000,ffff<http://pws.prserv.net/hilary<<\n",
      "\n",
      ">> \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">__________________________________________________\n",
      "\n",
      ">Do You Yahoo!?\n",
      "\n",
      ">Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "\n",
      ">http://greetings.yahoo.com\n",
      "\n",
      ">------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "\n",
      ">Buy Stock for $4.\n",
      "\n",
      ">No Minimums.\n",
      "\n",
      ">FREE Money 2002.\n",
      "\n",
      ">http://us.click.yahoo.com/BgmYkB/VovDAA/ySSFAA/NoOolB/TM\n",
      "\n",
      ">---------------------------------------------------------------------~->\n",
      "\n",
      ">\n",
      "\n",
      ">To unsubscribe from this group, send an email to:\n",
      "\n",
      ">emca-unsubscribe@egroups.com\n",
      "\n",
      ">\n",
      "\n",
      "> \n",
      "\n",
      ">\n",
      "\n",
      ">Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/ \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood1.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood2.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood3.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      "(Mr.) Hilary Smith\n",
      "\n",
      "712 Bomar St.\n",
      "\n",
      "Houston TX  77006-1316\n",
      "\n",
      "713.523.0559\n",
      "\n",
      "Fax: 713.523.3928\n",
      "\n",
      "Website:\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8b165749",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What a neighborhood!!  The pictures were wonderful.  I knew I wouldn't have any trouble printing the .jpg files, but I was surprised how easy it was to print from the .pdf file.  Each picture was a page.  I'm explaining that this is a test case: we want to see if District D can solve problems for us.  Ada Edwards has a great idea to set up a little neighborhood office next to the storefront police station.  I'm willing to help, but only if we can get results.  We don't need a public relations gimmick.\n",
      "\n",
      "\n",
      "Hilary\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "At 03:03 PM 3/1/2002 -0800, you wrote:\n",
      "\n",
      ">i took 3 pics\n",
      "\n",
      ">michel\n",
      "\n",
      ">--- Hilary Smith < wrote:\n",
      "\n",
      ">> I just took some pictures of Lake Bomar using old\n",
      "\n",
      ">> fashioned film which we\n",
      "\n",
      ">> probably won't develop for weeks.  If someone has a\n",
      "\n",
      ">> digital camera and\n",
      "\n",
      ">> could e-mail me a couple of pictures, I could print\n",
      "\n",
      ">> them and take them to\n",
      "\n",
      ">> Ada Edwards' office Monday.\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Hilary\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> At 02:35 PM 3/1/2002 -0600, you wrote: \n",
      "\n",
      ">> \n",
      "\n",
      ">> >>>>\n",
      "\n",
      ">> \n",
      "\n",
      ">> <anyone interested in seeing how swampy the\n",
      "\n",
      ">> lake gets - drive\n",
      "\n",
      ">> down Bomar toward taft and you'll see the \"site\" of\n",
      "\n",
      ">> discussion. A little\n",
      "\n",
      ">> rain goes a long way...\n",
      "\n",
      ">> \n",
      "\n",
      ">> Nathalie\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<0000,3333,9999<<Yahoo!\n",
      "\n",
      ">> Groups\n",
      "\n",
      ">> Sponsor<<<  \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<arial<<ADVERTISEMENT\n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> <\n",
      "\n",
      ">> \n",
      "\n",
      ">> To unsubscribe from this group, send an email to:\n",
      "\n",
      ">> \n",
      "\n",
      ">> emca-unsubscribe@egroups.com\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Your use of Yahoo! Groups is subject to the\n",
      "\n",
      ">> <<<Yahoo! Terms of\n",
      "\n",
      ">> Service.   \n",
      "\n",
      ">> \n",
      "\n",
      ">> <<<<<<<<<<<<<<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> (Mr.) Hilary Smith\n",
      "\n",
      ">> \n",
      "\n",
      ">> 712 Bomar St.\n",
      "\n",
      ">> \n",
      "\n",
      ">> Houston TX  77006-1316\n",
      "\n",
      ">> \n",
      "\n",
      ">> 713.523.0559\n",
      "\n",
      ">> \n",
      "\n",
      ">> Fax: 713.523.3928\n",
      "\n",
      ">> \n",
      "\n",
      ">> Website:\n",
      "\n",
      ">>\n",
      "\n",
      "><<<0000,0000,ffff<http://pws.prserv.net/hilary<<\n",
      "\n",
      ">> \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">__________________________________________________\n",
      "\n",
      ">Do You Yahoo!?\n",
      "\n",
      ">Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "\n",
      ">http://greetings.yahoo.com\n",
      "\n",
      ">------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "\n",
      ">Buy Stock for $4.\n",
      "\n",
      ">No Minimums.\n",
      "\n",
      ">FREE Money 2002.\n",
      "\n",
      ">http://us.click.yahoo.com/BgmYkB/VovDAA/ySSFAA/NoOolB/TM\n",
      "\n",
      ">---------------------------------------------------------------------~->\n",
      "\n",
      ">\n",
      "\n",
      ">To unsubscribe from this group, send an email to:\n",
      "\n",
      ">emca-unsubscribe@egroups.com\n",
      "\n",
      ">\n",
      "\n",
      "> \n",
      "\n",
      ">\n",
      "\n",
      ">Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/ \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood1.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood2.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood3.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      "(Mr.) Hilary Smith\n",
      "\n",
      "712 Bomar St.\n",
      "\n",
      "Houston TX  77006-1316\n",
      "\n",
      "713.523.0559\n",
      "\n",
      "Fax: 713.523.3928\n",
      "\n",
      "Website:\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(extract_content(\n",
    "    re_emails[re_emails['hasHtml'] == True]['content'].iloc[3], \n",
    "    stop_regexes=[\n",
    "        re.compile('\\s*Original Message'),\n",
    "        re.compile('\\s*Forwarded by'),\n",
    "        re.compile('^\\s?>?(From|To):\\s?', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2} (AM|PM)', re.I),\n",
    "        re.compile('\\son \\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}:\\d{2} (AM|PM)$', re.I)\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d10eae94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I love technology!!!!!\n",
      "Nathalie\n",
      "----- Original Message -----\n",
      "From: \"Michel Cahier\" \n",
      "To: \n",
      "Sent: Friday, March 01, 2002 5:03 PM\n",
      "Subject: *EMCA* here are some pics\n",
      "\n",
      "\n",
      "> i took 3 pics\n",
      "> michel\n",
      "> --- Hilary Smith  wrote:\n",
      "> > I just took some pictures of Lake Bomar using old\n",
      "> > fashioned film which we\n",
      "> > probably won't develop for weeks.  If someone has a\n",
      "> > digital camera and\n",
      "> > could e-mail me a couple of pictures, I could print\n",
      "> > them and take them to\n",
      "> > Ada Edwards' office Monday.\n",
      "> >\n",
      "> >\n",
      "> > Hilary\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> > At 02:35 PM 3/1/2002 -0600, you wrote:\n",
      "> >\n",
      "> > >>>>\n",
      "> >\n",
      "> > anyone interested in seeing how swampy the\n",
      "> > lake gets - drive\n",
      "> > down Bomar toward taft and you'll see the \"site\" of\n",
      "> > discussion. A little\n",
      "> > rain goes a long way...\n",
      "> >\n",
      "> > Nathalie\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> 0000,3333,9999Yahoo!\n",
      "> > Groups\n",
      "> > Sponsor\n",
      "> >\n",
      "> >\n",
      "> arialADVERTISEMENT\n",
      "> >\n",
      "> >\n",
      ">\n",
      "<\n",
      "> >\n",
      "> > \n",
      "> >\n",
      "> > To unsubscribe from this group, send an email to:\n",
      "> >\n",
      "> > emca-unsubscribe@egroups.com\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> > Your use of Yahoo! Groups is subject to the\n",
      "> > <Yahoo! Terms of\n",
      "> > Service.\n",
      "> >\n",
      "> > <<<<<<<<\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> >\n",
      "> > (Mr.) Hilary Smith\n",
      "> >\n",
      "> > 712 Bomar St.\n",
      "> >\n",
      "> > Houston TX  77006-1316\n",
      "> >\n",
      "> > 713.523.0559\n",
      "> >\n",
      "> > Fax: 713.523.3928\n",
      "> >\n",
      "> > Website:\n",
      "> >\n",
      ">\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary<\n",
      "/color>\n",
      "> >\n",
      ">\n",
      ">\n",
      "> __________________________________________________\n",
      "> Do You Yahoo!?\n",
      "> Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "> http://greetings.yahoo.com\n",
      ">\n",
      "> To unsubscribe from this group, send an email to:\n",
      "> emca-unsubscribe@egroups.com\n",
      ">\n",
      ">\n",
      ">\n",
      "> Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n",
      ">\n",
      ">\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "FREE COLLEGE MONEY\n",
      "CLICK HERE to search\n",
      "600,000 scholarships!\n",
      "http://us.click.yahoo.com/iZp8OC/4m7CAA/ySSFAA/NoOolB/TM\n",
      "---------------------------------------------------------------------~->\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "emca-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "707d92c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I love technology!!!!!\n",
      "Nathalie\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(extract_content(\n",
    "    re_emails[re_emails['hasHtml'] == True]['content'].iloc[4], \n",
    "    stop_regexes=[\n",
    "        re.compile('\\s*Original Message'),\n",
    "        re.compile('\\s*Forwarded by'),\n",
    "        re.compile('^\\s?>?(From|To):\\s?', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}', re.I),\n",
    "        re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2} (AM|PM)', re.I),\n",
    "        re.compile('\\son \\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}:\\d{2} (AM|PM)$', re.I)\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3147590b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create utility function to clean-up textual content\n",
    "\n",
    "def deal_equals_to(text):\n",
    "    \"\"\"\n",
    "    Function to remove \n",
    "    =09=09, =20 etc in the text \n",
    "    \"\"\"\n",
    "    condition1 = '[=]+\\d\\d'\n",
    "    condition2 = '[=]+\\d[a-zA-Z]\\?'\n",
    "    condition3 = '[=]+\\d[a-zA-Z]'\n",
    "    condition4 = '[=]+[ \\n]+'\n",
    "    \n",
    "    text = re.sub(condition1, '', str(text))\n",
    "    text = re.sub(condition2, '', str(text))\n",
    "    text = re.sub(condition3, '', str(text))\n",
    "    text = re.sub(condition4, '', str(text)).strip()\n",
    "    return text\n",
    "\n",
    "def clean_text(text):\n",
    "    \"\"\"\n",
    "    Clean text\n",
    "    \"\"\"\n",
    "    text = re.sub(r'<[^<]+?>', '', str(text)) # html <tags>\n",
    "    text = re.sub(r'\\[IMAGE\\]', '', str(text)) # html <tags>\n",
    "    text = text.replace('nbsp', '') # common in unprocessed html             \n",
    "    text = re.sub(r'^b\\s+', '', str(text))  # removes 'b' in the beginning of each text\n",
    "    text = re.sub(r'[>]+', '', str(text))\n",
    "    text = re.sub(r'[<]+', '', str(text))\n",
    "    text = re.sub(r'[_]+', '', str(text))\n",
    "    text = re.sub(r\"-{2,}\", \"\", str(text))\n",
    "    \n",
    "    #text = re.sub(r\"^[><\\-_*]+\", '', str(text)) # removes >,<,-,_,* in the beginning of each text \n",
    "    text = deal_equals_to(text) # removes =09=09, =20\n",
    "    # text = re.sub(r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$','',str(text)) # email addresses\n",
    "    # text = re.sub(r'(http|https|ftp)://[a-zA-Z0-9\\\\./]+', ' ', str(text)) # http/url regex\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d0806868",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What a neighborhood!!  The pictures were wonderful.  I knew I wouldn't have any trouble printing the .jpg files, but I was surprised how easy it was to print from the .pdf file.  Each picture was a page.  I'm explaining that this is a test case: we want to see if District D can solve problems for us.  Ada Edwards has a great idea to set up a little neighborhood office next to the storefront police station.  I'm willing to help, but only if we can get results.  We don't need a public relations gimmick.\n",
      "\n",
      "\n",
      "Hilary\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "At 03:03 PM 3/1/2002 -0800, you wrote:\n",
      "\n",
      ">i took 3 pics\n",
      "\n",
      ">michel\n",
      "\n",
      ">--- Hilary Smith < wrote:\n",
      "\n",
      ">> I just took some pictures of Lake Bomar using old\n",
      "\n",
      ">> fashioned film which we\n",
      "\n",
      ">> probably won't develop for weeks.  If someone has a\n",
      "\n",
      ">> digital camera and\n",
      "\n",
      ">> could e-mail me a couple of pictures, I could print\n",
      "\n",
      ">> them and take them to\n",
      "\n",
      ">> Ada Edwards' office Monday.\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Hilary\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> At 02:35 PM 3/1/2002 -0600, you wrote: \n",
      "\n",
      ">> \n",
      "\n",
      ">> >>>>\n",
      "\n",
      ">> \n",
      "\n",
      ">> <anyone interested in seeing how swampy the\n",
      "\n",
      ">> lake gets - drive\n",
      "\n",
      ">> down Bomar toward taft and you'll see the \"site\" of\n",
      "\n",
      ">> discussion. A little\n",
      "\n",
      ">> rain goes a long way...\n",
      "\n",
      ">> \n",
      "\n",
      ">> Nathalie\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<0000,3333,9999<<Yahoo!\n",
      "\n",
      ">> Groups\n",
      "\n",
      ">> Sponsor<<<  \n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<arial<<ADVERTISEMENT\n",
      "\n",
      ">> \n",
      "\n",
      ">>\n",
      "\n",
      "><<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> <\n",
      "\n",
      ">> \n",
      "\n",
      ">> To unsubscribe from this group, send an email to:\n",
      "\n",
      ">> \n",
      "\n",
      ">> emca-unsubscribe@egroups.com\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> Your use of Yahoo! Groups is subject to the\n",
      "\n",
      ">> <<<Yahoo! Terms of\n",
      "\n",
      ">> Service.   \n",
      "\n",
      ">> \n",
      "\n",
      ">> <<<<<<<<<<<<<<<<<\n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> \n",
      "\n",
      ">> (Mr.) Hilary Smith\n",
      "\n",
      ">> \n",
      "\n",
      ">> 712 Bomar St.\n",
      "\n",
      ">> \n",
      "\n",
      ">> Houston TX  77006-1316\n",
      "\n",
      ">> \n",
      "\n",
      ">> 713.523.0559\n",
      "\n",
      ">> \n",
      "\n",
      ">> Fax: 713.523.3928\n",
      "\n",
      ">> \n",
      "\n",
      ">> Website:\n",
      "\n",
      ">>\n",
      "\n",
      "><<<0000,0000,ffff<http://pws.prserv.net/hilary<<\n",
      "\n",
      ">> \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">__________________________________________________\n",
      "\n",
      ">Do You Yahoo!?\n",
      "\n",
      ">Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "\n",
      ">http://greetings.yahoo.com\n",
      "\n",
      ">------------------------ Yahoo! Groups Sponsor ---------------------~-->\n",
      "\n",
      ">Buy Stock for $4.\n",
      "\n",
      ">No Minimums.\n",
      "\n",
      ">FREE Money 2002.\n",
      "\n",
      ">http://us.click.yahoo.com/BgmYkB/VovDAA/ySSFAA/NoOolB/TM\n",
      "\n",
      ">---------------------------------------------------------------------~->\n",
      "\n",
      ">\n",
      "\n",
      ">To unsubscribe from this group, send an email to:\n",
      "\n",
      ">emca-unsubscribe@egroups.com\n",
      "\n",
      ">\n",
      "\n",
      "> \n",
      "\n",
      ">\n",
      "\n",
      ">Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/ \n",
      "\n",
      ">\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood1.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood2.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      ">Attachment Converted: \"c:\\eudora\\attach\\flood3.JPG\"\n",
      "\n",
      ">\n",
      "\n",
      "(Mr.) Hilary Smith\n",
      "\n",
      "712 Bomar St.\n",
      "\n",
      "Houston TX  77006-1316\n",
      "\n",
      "713.523.0559\n",
      "\n",
      "Fax: 713.523.3928\n",
      "\n",
      "Website:\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['hasHtml'] == True]['content'].iloc[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "cba7ebd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What a neighborhood!!  The pictures were wonderful.  I knew I wouldn't have any trouble printing the .jpg files, but I was surprised how easy it was to print from the .pdf file.  Each picture was a page.  I'm explaining that this is a test case: we want to see if District D can solve problems for us.  Ada Edwards has a great idea to set up a little neighborhood office next to the storefront police station.  I'm willing to help, but only if we can get results.  We don't need a public relations gimmick.\n",
      "\n",
      "\n",
      "Hilary\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "At 03:03 PM 3/1/2002 -0800, you wrote:\n",
      "\n",
      "i took 3 pics\n",
      "\n",
      "michel\n",
      "\n",
      " Hilary Smith  I just took some pictures of Lake Bomar using old\n",
      "\n",
      " fashioned film which we\n",
      "\n",
      " probably won't develop for weeks.  If someone has a\n",
      "\n",
      " digital camera and\n",
      "\n",
      " could e-mail me a couple of pictures, I could print\n",
      "\n",
      " them and take them to\n",
      "\n",
      " Ada Edwards' office Monday.\n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " Hilary\n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " At 02:35 PM 3/1/2002 -0600, you wrote: \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      "  lake gets - drive\n",
      "\n",
      " down Bomar toward taft and you'll see the \"site\" of\n",
      "\n",
      " discussion. A little\n",
      "\n",
      " rain goes a long way...\n",
      "\n",
      " \n",
      "\n",
      " Nathalie\n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      "\n",
      "\n",
      "0000,3333,9999 Groups\n",
      "\n",
      " Sponsor \n",
      "\n",
      "\n",
      "\n",
      "arial \n",
      "\n",
      "\n",
      "\n",
      " \n",
      "\n",
      "  \n",
      "\n",
      " To unsubscribe from this group, send an email to:\n",
      "\n",
      " \n",
      "\n",
      " emca-unsubscribe@egroups.com\n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " Your use of Yahoo! Groups is subject to the\n",
      "\n",
      "  Service.   \n",
      "\n",
      " \n",
      "\n",
      "  \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " (Mr.) Hilary Smith\n",
      "\n",
      " \n",
      "\n",
      " 712 Bomar St.\n",
      "\n",
      " \n",
      "\n",
      " Houston TX  77006-1316\n",
      "\n",
      " \n",
      "\n",
      " 713.523.0559\n",
      "\n",
      " \n",
      "\n",
      " Fax: 713.523.3928\n",
      "\n",
      " \n",
      "\n",
      " Website:\n",
      "\n",
      "\n",
      "\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Do You Yahoo!?\n",
      "\n",
      "Yahoo! Greetings - Send FREE e-cards for every occasion!\n",
      "\n",
      "http://greetings.yahoo.com\n",
      "\n",
      " Yahoo! Groups Sponsor ~\n",
      "\n",
      "Buy Stock for $4.\n",
      "\n",
      "No Minimums.\n",
      "\n",
      "FREE Money 2002.\n",
      "\n",
      "http://us.click.yahoo.com/BgmYkB/VovDAA/ySSFAA/NoOolB/TM\n",
      "\n",
      "~-\n",
      "\n",
      "\n",
      "\n",
      "To unsubscribe from this group, send an email to:\n",
      "\n",
      "emca-unsubscribe@egroups.com\n",
      "\n",
      "\n",
      "\n",
      " \n",
      "\n",
      "\n",
      "\n",
      "Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/ \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Attachment Converted: \"c:\\eudora\\attach\\flood1.JPG\"\n",
      "\n",
      "\n",
      "\n",
      "Attachment Converted: \"c:\\eudora\\attach\\flood2.JPG\"\n",
      "\n",
      "\n",
      "\n",
      "Attachment Converted: \"c:\\eudora\\attach\\flood3.JPG\"\n",
      "\n",
      "\n",
      "\n",
      "(Mr.) Hilary Smith\n",
      "\n",
      "712 Bomar St.\n",
      "\n",
      "Houston TX  77006-1316\n",
      "\n",
      "713.523.0559\n",
      "\n",
      "Fax: 713.523.3928\n",
      "\n",
      "Website:\n",
      "0000,0000,ffffhttp://pws.prserv.net/hilary\n"
     ]
    }
   ],
   "source": [
    "print(clean_text(re_emails[re_emails['hasHtml'] == True]['content'].iloc[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "009e46a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10003</th>\n",
       "      <td>&lt;24921653.1075854800772.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Fwd: EMAZING Recipe of the Day - Linguine ...</td>\n",
       "      <td>you can cook this for me!\\n\\n\\n\\n\\nkb  on 01/1...</td>\n",
       "      <td>961</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10205</th>\n",
       "      <td>&lt;26468961.1075854711777.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Fantast Basketball DRAFT TODAY @ 10:30</td>\n",
       "      <td>html is better but either one works\\n\\n\\nTo: E...</td>\n",
       "      <td>345</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12052</th>\n",
       "      <td>&lt;973227.1075854738455.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Fwd: EMAZING Recipe of the Day - Linguine ...</td>\n",
       "      <td>you can cook this for me!\\n\\n\\n\\n\\nkb  on 01/1...</td>\n",
       "      <td>961</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12780</th>\n",
       "      <td>&lt;15230645.1075854598702.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Fantast Basketball DRAFT TODAY @ 10:30</td>\n",
       "      <td>html is better but either one works\\n\\n\\nTo: E...</td>\n",
       "      <td>345</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16121</th>\n",
       "      <td>&lt;617555.1075854685433.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Fantast Basketball DRAFT TODAY @ 10:30</td>\n",
       "      <td>html is better but either one works\\n\\n\\nTo: E...</td>\n",
       "      <td>345</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16171</th>\n",
       "      <td>&lt;27123905.1075854776897.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Fwd: EMAZING Recipe of the Day - Linguine ...</td>\n",
       "      <td>you can cook this for me!\\n\\n\\n\\n\\nkb  on 01/1...</td>\n",
       "      <td>961</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84720</th>\n",
       "      <td>&lt;31909535.1075854556083.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Conference Call and Web Presentation: Vola...</td>\n",
       "      <td>Angela, \\n\\nIs Dave supposed to be involved in...</td>\n",
       "      <td>851</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85623</th>\n",
       "      <td>&lt;7325102.1075854499646.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Conference Call and Web Presentation: Vola...</td>\n",
       "      <td>Angela, \\n\\nIs Dave supposed to be involved in...</td>\n",
       "      <td>851</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87379</th>\n",
       "      <td>&lt;28598123.1075854536470.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Conference Call and Web Presentation: Vola...</td>\n",
       "      <td>Angela, \\n\\nIs Dave supposed to be involved in...</td>\n",
       "      <td>851</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193282</th>\n",
       "      <td>&lt;5740016.1075856998375.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Nurit,\\n\\nWe shall schedule a phone interview ...</td>\n",
       "      <td>705</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194035</th>\n",
       "      <td>&lt;16877338.1075856520504.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>John,\\n\\nHTML version of Enron article:\\n\\n\\nV...</td>\n",
       "      <td>202</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194540</th>\n",
       "      <td>&lt;27602311.1075856764897.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Thank you.\\nNurit\\n\\n*************************...</td>\n",
       "      <td>888</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194543</th>\n",
       "      <td>&lt;32822837.1075856765002.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Nurit,\\n\\nWe shall schedule a phone interview ...</td>\n",
       "      <td>705</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195969</th>\n",
       "      <td>&lt;11565254.1075856221015.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>Vince,\\n\\nI got the zipped version fine.  Than...</td>\n",
       "      <td>443</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195972</th>\n",
       "      <td>&lt;31193049.1075856221082.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>John,\\n\\nHTML version of Enron article:\\n\\n\\nV...</td>\n",
       "      <td>202</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207069</th>\n",
       "      <td>&lt;6392625.1075856399244.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>John,\\n\\nHTML version of Enron article:\\n\\n\\nV...</td>\n",
       "      <td>202</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207078</th>\n",
       "      <td>&lt;13005045.1075856399475.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>Vince,\\n\\nI got the zipped version fine.  Than...</td>\n",
       "      <td>443</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209078</th>\n",
       "      <td>&lt;24269738.1075856807958.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Nurit,\\n\\nWe shall schedule a phone interview ...</td>\n",
       "      <td>705</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209087</th>\n",
       "      <td>&lt;11616844.1075856808187.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Thank you.\\nNurit\\n\\n*************************...</td>\n",
       "      <td>888</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212549</th>\n",
       "      <td>&lt;6395000.1075857068606.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Thank you.\\nNurit\\n\\n*************************...</td>\n",
       "      <td>888</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216936</th>\n",
       "      <td>&lt;7596377.1075856941484.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Job application</td>\n",
       "      <td>Nurit,\\n\\nWe shall schedule a phone interview ...</td>\n",
       "      <td>705</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217689</th>\n",
       "      <td>&lt;4453337.1075856454190.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>John,\\n\\nHTML version of Enron article:\\n\\n\\nV...</td>\n",
       "      <td>202</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218433</th>\n",
       "      <td>&lt;17080865.1075856633449.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Latest revision</td>\n",
       "      <td>Vince,\\n\\nI got the zipped version fine.  Than...</td>\n",
       "      <td>443</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218821</th>\n",
       "      <td>&lt;29468798.1075846168582.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Computer Based Communications Training Ini...</td>\n",
       "      <td>In my interview I stressed that a big part of ...</td>\n",
       "      <td>707</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223176</th>\n",
       "      <td>&lt;22037287.1075846240907.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Computer Based Communications Training Ini...</td>\n",
       "      <td>In my interview I stressed that a big part of ...</td>\n",
       "      <td>707</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228917</th>\n",
       "      <td>&lt;21279790.1075851018673.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Computer Based Communications Training Ini...</td>\n",
       "      <td>In my interview I stressed that a big part of ...</td>\n",
       "      <td>707</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229295</th>\n",
       "      <td>&lt;10027637.1075846299722.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Computer Based Communications Training Ini...</td>\n",
       "      <td>In my interview I stressed that a big part of ...</td>\n",
       "      <td>707</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241741</th>\n",
       "      <td>&lt;16342743.1075846378782.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Computer Based Communications Training Ini...</td>\n",
       "      <td>In my interview I stressed that a big part of ...</td>\n",
       "      <td>707</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444641</th>\n",
       "      <td>&lt;26817710.1075841911399.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: a beer</td>\n",
       "      <td>Sure - let's do the Lucky Labrador, if it work...</td>\n",
       "      <td>380</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444663</th>\n",
       "      <td>&lt;23561802.1075841911445.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: a beer</td>\n",
       "      <td>Okay - I won't be able to get out of here befo...</td>\n",
       "      <td>215</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447289</th>\n",
       "      <td>&lt;2279382.1075841704148.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: a beer</td>\n",
       "      <td>Okay - I won't be able to get out of here befo...</td>\n",
       "      <td>215</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447292</th>\n",
       "      <td>&lt;7538148.1075841704244.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: a beer</td>\n",
       "      <td>Sure - let's do the Lucky Labrador, if it work...</td>\n",
       "      <td>380</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452942</th>\n",
       "      <td>&lt;2783676.1075841908109.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: a beer</td>\n",
       "      <td>Okay - I won't be able to get out of here befo...</td>\n",
       "      <td>215</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452944</th>\n",
       "      <td>&lt;16916790.1075841908156.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: a beer</td>\n",
       "      <td>Sure - let's do the Lucky Labrador, if it work...</td>\n",
       "      <td>380</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>458188</th>\n",
       "      <td>&lt;26353469.1075859896099.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: More FAQ's</td>\n",
       "      <td>I'm afraid I'm still the right guy.  My commen...</td>\n",
       "      <td>209</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466616</th>\n",
       "      <td>&lt;8286300.1075860062425.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: More FAQ's</td>\n",
       "      <td>I'm afraid I'm still the right guy.  My commen...</td>\n",
       "      <td>209</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "10003   <24921653.1075854800772.JavaMail.evans@thyme>   \n",
       "10205   <26468961.1075854711777.JavaMail.evans@thyme>   \n",
       "12052     <973227.1075854738455.JavaMail.evans@thyme>   \n",
       "12780   <15230645.1075854598702.JavaMail.evans@thyme>   \n",
       "16121     <617555.1075854685433.JavaMail.evans@thyme>   \n",
       "16171   <27123905.1075854776897.JavaMail.evans@thyme>   \n",
       "84720   <31909535.1075854556083.JavaMail.evans@thyme>   \n",
       "85623    <7325102.1075854499646.JavaMail.evans@thyme>   \n",
       "87379   <28598123.1075854536470.JavaMail.evans@thyme>   \n",
       "193282   <5740016.1075856998375.JavaMail.evans@thyme>   \n",
       "194035  <16877338.1075856520504.JavaMail.evans@thyme>   \n",
       "194540  <27602311.1075856764897.JavaMail.evans@thyme>   \n",
       "194543  <32822837.1075856765002.JavaMail.evans@thyme>   \n",
       "195969  <11565254.1075856221015.JavaMail.evans@thyme>   \n",
       "195972  <31193049.1075856221082.JavaMail.evans@thyme>   \n",
       "207069   <6392625.1075856399244.JavaMail.evans@thyme>   \n",
       "207078  <13005045.1075856399475.JavaMail.evans@thyme>   \n",
       "209078  <24269738.1075856807958.JavaMail.evans@thyme>   \n",
       "209087  <11616844.1075856808187.JavaMail.evans@thyme>   \n",
       "212549   <6395000.1075857068606.JavaMail.evans@thyme>   \n",
       "216936   <7596377.1075856941484.JavaMail.evans@thyme>   \n",
       "217689   <4453337.1075856454190.JavaMail.evans@thyme>   \n",
       "218433  <17080865.1075856633449.JavaMail.evans@thyme>   \n",
       "218821  <29468798.1075846168582.JavaMail.evans@thyme>   \n",
       "223176  <22037287.1075846240907.JavaMail.evans@thyme>   \n",
       "228917  <21279790.1075851018673.JavaMail.evans@thyme>   \n",
       "229295  <10027637.1075846299722.JavaMail.evans@thyme>   \n",
       "241741  <16342743.1075846378782.JavaMail.evans@thyme>   \n",
       "444641  <26817710.1075841911399.JavaMail.evans@thyme>   \n",
       "444663  <23561802.1075841911445.JavaMail.evans@thyme>   \n",
       "447289   <2279382.1075841704148.JavaMail.evans@thyme>   \n",
       "447292   <7538148.1075841704244.JavaMail.evans@thyme>   \n",
       "452942   <2783676.1075841908109.JavaMail.evans@thyme>   \n",
       "452944  <16916790.1075841908156.JavaMail.evans@thyme>   \n",
       "458188  <26353469.1075859896099.JavaMail.evans@thyme>   \n",
       "466616   <8286300.1075860062425.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "10003   Re: Fwd: EMAZING Recipe of the Day - Linguine ...   \n",
       "10205          Re: Fantast Basketball DRAFT TODAY @ 10:30   \n",
       "12052   Re: Fwd: EMAZING Recipe of the Day - Linguine ...   \n",
       "12780          Re: Fantast Basketball DRAFT TODAY @ 10:30   \n",
       "16121          Re: Fantast Basketball DRAFT TODAY @ 10:30   \n",
       "16171   Re: Fwd: EMAZING Recipe of the Day - Linguine ...   \n",
       "84720   Re: Conference Call and Web Presentation: Vola...   \n",
       "85623   Re: Conference Call and Web Presentation: Vola...   \n",
       "87379   Re: Conference Call and Web Presentation: Vola...   \n",
       "193282                                Re: Job application   \n",
       "194035                                Re: Latest revision   \n",
       "194540                                Re: Job application   \n",
       "194543                                Re: Job application   \n",
       "195969                                Re: Latest revision   \n",
       "195972                                Re: Latest revision   \n",
       "207069                                Re: Latest revision   \n",
       "207078                                Re: Latest revision   \n",
       "209078                                Re: Job application   \n",
       "209087                                Re: Job application   \n",
       "212549                                Re: Job application   \n",
       "216936                                Re: Job application   \n",
       "217689                                Re: Latest revision   \n",
       "218433                                Re: Latest revision   \n",
       "218821  Re: Computer Based Communications Training Ini...   \n",
       "223176  Re: Computer Based Communications Training Ini...   \n",
       "228917  Re: Computer Based Communications Training Ini...   \n",
       "229295  Re: Computer Based Communications Training Ini...   \n",
       "241741  Re: Computer Based Communications Training Ini...   \n",
       "444641                                         Re: a beer   \n",
       "444663                                         Re: a beer   \n",
       "447289                                         Re: a beer   \n",
       "447292                                         Re: a beer   \n",
       "452942                                         Re: a beer   \n",
       "452944                                         Re: a beer   \n",
       "458188                                     Re: More FAQ's   \n",
       "466616                                     Re: More FAQ's   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "10003   you can cook this for me!\\n\\n\\n\\n\\nkb  on 01/1...      961    False  \n",
       "10205   html is better but either one works\\n\\n\\nTo: E...      345    False  \n",
       "12052   you can cook this for me!\\n\\n\\n\\n\\nkb  on 01/1...      961    False  \n",
       "12780   html is better but either one works\\n\\n\\nTo: E...      345    False  \n",
       "16121   html is better but either one works\\n\\n\\nTo: E...      345    False  \n",
       "16171   you can cook this for me!\\n\\n\\n\\n\\nkb  on 01/1...      961    False  \n",
       "84720   Angela, \\n\\nIs Dave supposed to be involved in...      851    False  \n",
       "85623   Angela, \\n\\nIs Dave supposed to be involved in...      851    False  \n",
       "87379   Angela, \\n\\nIs Dave supposed to be involved in...      851    False  \n",
       "193282  Nurit,\\n\\nWe shall schedule a phone interview ...      705    False  \n",
       "194035  John,\\n\\nHTML version of Enron article:\\n\\n\\nV...      202    False  \n",
       "194540  Thank you.\\nNurit\\n\\n*************************...      888    False  \n",
       "194543  Nurit,\\n\\nWe shall schedule a phone interview ...      705    False  \n",
       "195969  Vince,\\n\\nI got the zipped version fine.  Than...      443    False  \n",
       "195972  John,\\n\\nHTML version of Enron article:\\n\\n\\nV...      202    False  \n",
       "207069  John,\\n\\nHTML version of Enron article:\\n\\n\\nV...      202    False  \n",
       "207078  Vince,\\n\\nI got the zipped version fine.  Than...      443    False  \n",
       "209078  Nurit,\\n\\nWe shall schedule a phone interview ...      705    False  \n",
       "209087  Thank you.\\nNurit\\n\\n*************************...      888    False  \n",
       "212549  Thank you.\\nNurit\\n\\n*************************...      888    False  \n",
       "216936  Nurit,\\n\\nWe shall schedule a phone interview ...      705    False  \n",
       "217689  John,\\n\\nHTML version of Enron article:\\n\\n\\nV...      202    False  \n",
       "218433  Vince,\\n\\nI got the zipped version fine.  Than...      443    False  \n",
       "218821  In my interview I stressed that a big part of ...      707    False  \n",
       "223176  In my interview I stressed that a big part of ...      707    False  \n",
       "228917  In my interview I stressed that a big part of ...      707    False  \n",
       "229295  In my interview I stressed that a big part of ...      707    False  \n",
       "241741  In my interview I stressed that a big part of ...      707    False  \n",
       "444641  Sure - let's do the Lucky Labrador, if it work...      380    False  \n",
       "444663  Okay - I won't be able to get out of here befo...      215    False  \n",
       "447289  Okay - I won't be able to get out of here befo...      215    False  \n",
       "447292  Sure - let's do the Lucky Labrador, if it work...      380    False  \n",
       "452942  Okay - I won't be able to get out of here befo...      215    False  \n",
       "452944  Sure - let's do the Lucky Labrador, if it work...      380    False  \n",
       "458188  I'm afraid I'm still the right guy.  My commen...      209    False  \n",
       "466616  I'm afraid I'm still the right guy.  My commen...      209    False  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_emails[re_emails['content'].str.contains('HTML')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "62c8bd30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you can cook this for me!\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "kb  on 01/12/2001 12:10:02 PM\n",
      "To: eric.bass@enron.com\n",
      "cc: =20\n",
      "Subject: Fwd: EMAZING Recipe of the Day - Linguine Puttanesca\n",
      "\n",
      "\n",
      "Hi E,\n",
      "\n",
      "Thought you might find this recipe appealing.  Have a\n",
      "great day.  LU-M\n",
      "--- EMAZING \n",
      "wrote:\n",
      "> Reply-to: \n",
      "> Date: Fri, 12 Jan 2001 10:25:32 GMT\n",
      "> Subject: EMAZING Recipe of the Day - Linguine\n",
      "> Puttanesca\n",
      "> From: EMAZING\n",
      "> \n",
      "> To: EMAZING \n",
      ">\n",
      "> EMAZING - http://www.emazing.com\n",
      ">\n",
      "> Proudly Presents\n",
      ">\n",
      "> The EMAZING Recipe of the Day - (TEXT)\n",
      "> Friday January 12, 2001\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      "> FREE $5 at HALF.com\n",
      "> Are you getting your Books, CDs, DVDs, & Video Games\n",
      "> for\n",
      "> HALF off? At Half.com get up to 50-90% off the list\n",
      "> price\n",
      "> everyday.  We=01,ve got The Matrix, American Pie, U2,\n",
      "> Dixie\n",
      "> Chicks and much more.  For a limited time, get $5\n",
      "> off your\n",
      "> first purchase of $10 or more.\n",
      "> http://click.emazing.com/ads/half/text5.html\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Linguine Puttanesca\n",
      ">\n",
      "> Serve this meatless entree with a salad and bread to\n",
      "> make a\n",
      "> satisfying meal.\n",
      ">\n",
      "> Serves 4\n",
      ">\n",
      "> 1 pound Linguine, cooked, drained, and rinsed\n",
      "> 1/3 cup Olive oil\n",
      "> 3 cloves Garlic, minced\n",
      "> 1/4 teaspoon Crushed red pepper flake\n",
      "> 2 1/2 cups Tomatoes, diced\n",
      "> 1/2 cup Black olives (pitted), coarsely chopped\n",
      "> 2 Tablespoons Capers\n",
      "> 2 Tablespoons Basil, fresh, chopped\n",
      "> To taste Salt and pepper\n",
      ">\n",
      "> Heat the oil in a large skillet over low heat. Add\n",
      "> the\n",
      "> garlic and crushed pepper and heat for 3 minutes (do\n",
      "> not\n",
      "> brown). Add the tomatoes and cook for 5 minutes. Add\n",
      "> the\n",
      "> olives, capers, and basil and cook for 3 minutes\n",
      "> longer.\n",
      "> Toss in the cooked linguine. Serve warm.\n",
      ">\n",
      "> - Nick Sundberg\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Email a copy of this tip to a friend\n",
      ">\n",
      "http://www.emazing.com/cgi-bin/mtaf.cgi?date=3D01-12-2001&list_id=3Drecipe&=\n",
      "brand=3De\n",
      "mazing\n",
      ">\n",
      "> Browse the archives\n",
      "> http://www.emazing.com/archives/recipe\n",
      ">\n",
      "> Unsubscribe\n",
      "> http://www.emazing.com/unsub/recipe\n",
      ">\n",
      "> EMAZING Home\n",
      "> http://www.emazing.com\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> DEALS OF THE DAY\n",
      ">\n",
      "> RECIPE BOOK OF THE DAY\n",
      "> Gwen McKee: The Recipe Hall of Fame Cookbook\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/1893062082/emazing\n",
      ">\n",
      "> Conversion Tables\n",
      "> Click here to convert weights and measures between\n",
      "> Metric and U.S. Standards.\n",
      "> http://www.cambridgeculinary.com/html/equiv.html\n",
      ">\n",
      "> BOOK OF THE DAY\n",
      "> Brad Meltzer: The First Counsel\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/0446527289/emazing\n",
      ">\n",
      "> CD OF THE DAY\n",
      "> Tim McGraw: Greatest Hits\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B00004ZAZH/emazing\n",
      ">\n",
      "> GIFT OF THE DAY\n",
      "> Diamond Rio 600 32MB Digital Audio Player\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B00004SPUN/emazing\n",
      ">\n",
      "> MOVIE OF THE DAY (DVD)\n",
      "> Billy Crystal: When Harry Met Sally\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B00003CXDC/emazing\n",
      ">\n",
      "> MOVIE OF THE DAY (VHS)\n",
      "> Tobey Maguire: The Cider House Rules\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B0000507PC/emazing\n",
      ">\n",
      "> Another new year...\n",
      "> Another reason to send an EMAZING Greeting Card!\n",
      "> 2001 is going to be full of birthdays,\n",
      "> anniversaries, holidays and everyday occasions.\n",
      "> Don't miss out--send a FREE and fun EMAZING Greeting\n",
      "> Card today!\n",
      "> http://greetings.emazing.com/\n",
      ">\n",
      "> It's that time of the year again... make it easy on\n",
      "> yourself.\n",
      "> Click here to get TurboTax!\n",
      ">\n",
      "http://service.bfast.com/bfast/click?bfmid=3D32881117&siteid=3D36901868&bfp=\n",
      "age=3Dtur\n",
      "botax\n",
      ">\n",
      "> Need money for college?\n",
      "> FastWeb is the largest FREE scholarship search on\n",
      "> the Internet, with accurate, updated information on\n",
      "> more than 600,000 scholarships, grants, and\n",
      "> fellowships.\n",
      "> http://click.emazing.com/ads/fastweb/text2.html\n",
      ">\n",
      "> Get satellite television in only one step.\n",
      "> Click here for more information on FREE satellite\n",
      "> TV!\n",
      ">\n",
      "http://www.commission-junction.com/track/track.dll?AID=3D1203660&PID=3D5521=\n",
      "59&URL=3D\n",
      "http%3A%2F%2Fwww%2EskyboxTV%2Ecom\n",
      ">\n",
      "> Send Fresh Flowers and Gifts from FTD.COM Today!\n",
      "> Don't forget, Valentine's Day is Wednesday, February\n",
      "> 14th. BE A HERO AT FTD.com.\n",
      "> http://click.emazing.com/ads/ftd/flowers.html\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      "> Another new year...\n",
      "> Another reason to send an EMAZING Greeting Card!\n",
      "> 2001 is\n",
      "> going to be full of birthdays, anniversaries,\n",
      "> holidays and\n",
      "> everyday occasions. Don't miss out--send a FREE and\n",
      "> fun\n",
      "> EMAZING Greeting Card today!\n",
      "> http://greetings.emazing.com/\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Quote of the Day\n",
      ">\n",
      "> Some cause happiness wherever they go; others,\n",
      "> whenever they go.\n",
      "> - Oscar Wilde\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> EMAZING SUBSCRIBER SERVICES\n",
      ">\n",
      "> EMAZING Home\n",
      "> http://www.emazing.com/\n",
      ">\n",
      "> Advice and Living\n",
      "> http://www.emazing.com/advice/\n",
      ">\n",
      "> Brain Candy\n",
      "> http://www.emazing.com/brain/\n",
      ">\n",
      "> Business and Money\n",
      "> http://www.emazing.com/business/\n",
      ">\n",
      "> Comics\n",
      "> http://www.emazing.com/comics/\n",
      ">\n",
      "> Daybreakers\n",
      "> http://www.emazing.com/daybreakers/\n",
      ">\n",
      "> Entertainment\n",
      "> http://www.emazing.com/entertainment/\n",
      ">\n",
      "> Espa=0Fol\n",
      "> http://espanol.emazing.com/\n",
      ">\n",
      "> Games\n",
      "> http://games.emazing.com/\n",
      ">\n",
      "> Good Thoughts\n",
      "> http://www.emazing.com/goodthoughts/\n",
      ">\n",
      "> Grapevine\n",
      "> http://grapevine.emazing.com/\n",
      ">\n",
      "> Greetings\n",
      "> http://greetings.emazing.com/\n",
      ">\n",
      "> Health\n",
      "> http://www.emazing.com/health/\n",
      ">\n",
      "> Hobbies\n",
      "> http://www.emazing.com/hobbies/\n",
      ">\n",
      "> Horoscopes\n",
      "> http://www.emazing.com/horoscopes/\n",
      ">\n",
      "> Household Hints\n",
      "> http://www.emazing.com/household/\n",
      ">\n",
      "> Kitchen\n",
      "> http://www.emazing.com/kitchen/\n",
      ">\n",
      "> Music\n",
      "> http://www.emazing.com/music/\n",
      ">\n",
      "> Sports\n",
      "> http://www.emazing.com/sports/\n",
      ">\n",
      "> Style\n",
      "> http://www.emazing.com/style/\n",
      ">\n",
      "> Technology\n",
      "> http://www.emazing.com/technology/\n",
      ">\n",
      "> Travel\n",
      "> http://www.emazing.com/travel/\n",
      ">\n",
      "> Wireless\n",
      "> http://www.emazing.com/wireless/\n",
      ">\n",
      "> Unsubscribe\n",
      "> http://www.emazing.com/unsub/recipe\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Advertise\n",
      "> http://www.emazing.com/info/advertise.htm\n",
      ">\n",
      "> Archives\n",
      "> http://www.emazing.com/archives/\n",
      ">\n",
      "> Company information\n",
      "> http://www.emazing.com/info/\n",
      ">\n",
      "> Edit Your Profile\n",
      "> http://www.emazing.com/profile/\n",
      ">\n",
      "> Feedback\n",
      "> Got comments? Suggestions? Complaints?\n",
      "> Send us an email to feedback@emazing.com\n",
      ">\n",
      "> Help\n",
      "> http://www.emazing.com/info/help.htm\n",
      ">\n",
      "> Jobs\n",
      "> http://www.emazing.com/info/jobs.htm\n",
      ">\n",
      "> Privacy Policy\n",
      "> http://www.emazing.com/info/privacy.htm\n",
      ">\n",
      "> Switch To HTML Format\n",
      "> http://www.emazing.com/switchtohtml.htm\n",
      ">\n",
      "> Terms & Conditions\n",
      "> http://www.emazing.com/info/terms.htm\n",
      ">\n",
      "> EMAZING(tm) is a service of EMAZING, INC.\n",
      "> Copyright 2001. All Rights Reserved.\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> EMAZING - Whatever you're into, we email it to you.\n",
      "> FREE!\n",
      "> http://www.emazing.com\n",
      ">\n",
      ">\n",
      ">\n",
      "\n",
      "\n",
      "__________________________________________________\n",
      "Do You Yahoo!?\n",
      "Yahoo! Photos - Share your holiday photos online!\n",
      "http://photos.yahoo.com/\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['content'].str.contains('HTML')]['content'].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1a78d517",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html is better but either one works\n",
      "\n",
      "\n",
      "To: Eric Bass/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Re: Fantast Basketball DRAFT TODAY @ 10:30  \n",
      "\n",
      "Rude Boys are in da house... Can't remember... Do we choose Java or HTML...\n",
      "\n",
      "\n",
      "   \n",
      "\tEnron North America Corp.\n",
      "\t\n",
      "\tFrom:  Eric Bass                           10/31/2000 08:25 AM\n",
      "\t\n",
      "\n",
      "To: David Baumbach/HOU/ECT@ECT, Phillip M Love/HOU/ECT@ECT, Bryan \n",
      "Hull/HOU/ECT@ECT, Harry Bucalo/HOU/ECT@ECT, O'Neal D Winfree/HOU/ECT@ECT, \n",
      "Michael Walters/HOU/ECT@ECT, Matthew Lenhart/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Fantast Basketball DRAFT TODAY @ 10:30\n",
      "\n",
      "Tell me if you are in on the draft at 10:30.  To draft you need to go to the \n",
      "website and click on draft - go to the draft room.\n",
      "---------------------- Forwarded by Eric Bass/HOU/ECT on 10/31/2000 08:23 AM \n",
      "---------------------------\n",
      "   \n",
      "\tEnron North America Corp.\n",
      "\t\n",
      "\tFrom:  Eric Bass                           10/31/2000 08:21 AM\n",
      "\t\n",
      "\n",
      "To: David Baumbach/HOU/ECT@ECT, Phillip M Love/HOU/ECT@ECT, Bryan \n",
      "Hull/HOU/ECT@ECT, Harry Bucalo/HOU/ECT@ECT, O'Neal D Winfree/HOU/ECT@ECT, \n",
      "Michael Walters/HOU/ECT@ECT, Matthew Lenhart/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Fantast Basketball DRAFT TODAY @ 10:30\n",
      "\n",
      "Just as clarification - we start 6 players and have weekly scoring in haed to \n",
      "head matchups.  You can chage your lineup before each day's worth of action.  \n",
      "Your sixth man can be any position.  The draft order will be determind at \n",
      "10:30.\n",
      "---------------------- Forwarded by Eric Bass/HOU/ECT on 10/31/2000 08:19 AM \n",
      "---------------------------\n",
      "   \n",
      "\tEnron North America Corp.\n",
      "\t\n",
      "\tFrom:  Eric Bass                           10/31/2000 08:01 AM\n",
      "\t\n",
      "\n",
      "To: David Baumbach/HOU/ECT@ECT, Phillip M Love/HOU/ECT@ECT, Bryan \n",
      "Hull/HOU/ECT@ECT, Harry Bucalo/HOU/ECT@ECT, O'Neal D Winfree/HOU/ECT@ECT, \n",
      "Michael Walters/HOU/ECT@ECT, Matthew Lenhart/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Fantast Basketball DRAFT TODAY @ 10:30\n",
      "\n",
      "Since the NBA season begins tonight, we need to have the fantasy basketball \n",
      "draft today @ 10:30 AM. Tell me if this is ok.  If you do not remember the \n",
      "rules, go to the link below and scroll down to the bottom of the page to the \n",
      "rules link.\n",
      "\n",
      "The Academy will roll once again!\n",
      "\n",
      "And by the way plove, it is last day for everyone.  You can take 1 minute out \n",
      "of every half hour today to make a pick.  Everyone else has agreed.  Besides \n",
      "it won't be as much fun without my whipping boy again.\n",
      "\n",
      "basketball.commissioner.com/efba\n",
      " \n",
      "I never heard anything back from Walker's Westies so The Rounders are taking \n",
      "his spot.\n",
      "\n",
      "-Eric\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['content'].str.contains('HTML')]['content'].iloc[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5ba0a11c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you can cook this for me!\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "kb  on 01/12/2001 12:10:02 PM\n",
      "To: eric.bass@enron.com\n",
      "cc: =20\n",
      "Subject: Fwd: EMAZING Recipe of the Day - Linguine Puttanesca\n",
      "\n",
      "\n",
      "Hi E,\n",
      "\n",
      "Thought you might find this recipe appealing.  Have a\n",
      "great day.  LU-M\n",
      "--- EMAZING \n",
      "wrote:\n",
      "> Reply-to: \n",
      "> Date: Fri, 12 Jan 2001 10:25:32 GMT\n",
      "> Subject: EMAZING Recipe of the Day - Linguine\n",
      "> Puttanesca\n",
      "> From: EMAZING\n",
      "> \n",
      "> To: EMAZING \n",
      ">\n",
      "> EMAZING - http://www.emazing.com\n",
      ">\n",
      "> Proudly Presents\n",
      ">\n",
      "> The EMAZING Recipe of the Day - (TEXT)\n",
      "> Friday January 12, 2001\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      "> FREE $5 at HALF.com\n",
      "> Are you getting your Books, CDs, DVDs, & Video Games\n",
      "> for\n",
      "> HALF off? At Half.com get up to 50-90% off the list\n",
      "> price\n",
      "> everyday.  We=01,ve got The Matrix, American Pie, U2,\n",
      "> Dixie\n",
      "> Chicks and much more.  For a limited time, get $5\n",
      "> off your\n",
      "> first purchase of $10 or more.\n",
      "> http://click.emazing.com/ads/half/text5.html\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Linguine Puttanesca\n",
      ">\n",
      "> Serve this meatless entree with a salad and bread to\n",
      "> make a\n",
      "> satisfying meal.\n",
      ">\n",
      "> Serves 4\n",
      ">\n",
      "> 1 pound Linguine, cooked, drained, and rinsed\n",
      "> 1/3 cup Olive oil\n",
      "> 3 cloves Garlic, minced\n",
      "> 1/4 teaspoon Crushed red pepper flake\n",
      "> 2 1/2 cups Tomatoes, diced\n",
      "> 1/2 cup Black olives (pitted), coarsely chopped\n",
      "> 2 Tablespoons Capers\n",
      "> 2 Tablespoons Basil, fresh, chopped\n",
      "> To taste Salt and pepper\n",
      ">\n",
      "> Heat the oil in a large skillet over low heat. Add\n",
      "> the\n",
      "> garlic and crushed pepper and heat for 3 minutes (do\n",
      "> not\n",
      "> brown). Add the tomatoes and cook for 5 minutes. Add\n",
      "> the\n",
      "> olives, capers, and basil and cook for 3 minutes\n",
      "> longer.\n",
      "> Toss in the cooked linguine. Serve warm.\n",
      ">\n",
      "> - Nick Sundberg\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Email a copy of this tip to a friend\n",
      ">\n",
      "http://www.emazing.com/cgi-bin/mtaf.cgi?date=3D01-12-2001&list_id=3Drecipe&=\n",
      "brand=3De\n",
      "mazing\n",
      ">\n",
      "> Browse the archives\n",
      "> http://www.emazing.com/archives/recipe\n",
      ">\n",
      "> Unsubscribe\n",
      "> http://www.emazing.com/unsub/recipe\n",
      ">\n",
      "> EMAZING Home\n",
      "> http://www.emazing.com\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> DEALS OF THE DAY\n",
      ">\n",
      "> RECIPE BOOK OF THE DAY\n",
      "> Gwen McKee: The Recipe Hall of Fame Cookbook\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/1893062082/emazing\n",
      ">\n",
      "> Conversion Tables\n",
      "> Click here to convert weights and measures between\n",
      "> Metric and U.S. Standards.\n",
      "> http://www.cambridgeculinary.com/html/equiv.html\n",
      ">\n",
      "> BOOK OF THE DAY\n",
      "> Brad Meltzer: The First Counsel\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/0446527289/emazing\n",
      ">\n",
      "> CD OF THE DAY\n",
      "> Tim McGraw: Greatest Hits\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B00004ZAZH/emazing\n",
      ">\n",
      "> GIFT OF THE DAY\n",
      "> Diamond Rio 600 32MB Digital Audio Player\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B00004SPUN/emazing\n",
      ">\n",
      "> MOVIE OF THE DAY (DVD)\n",
      "> Billy Crystal: When Harry Met Sally\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B00003CXDC/emazing\n",
      ">\n",
      "> MOVIE OF THE DAY (VHS)\n",
      "> Tobey Maguire: The Cider House Rules\n",
      ">\n",
      "http://www.amazon.com/exec/obidos/ASIN/B0000507PC/emazing\n",
      ">\n",
      "> Another new year...\n",
      "> Another reason to send an EMAZING Greeting Card!\n",
      "> 2001 is going to be full of birthdays,\n",
      "> anniversaries, holidays and everyday occasions.\n",
      "> Don't miss out--send a FREE and fun EMAZING Greeting\n",
      "> Card today!\n",
      "> http://greetings.emazing.com/\n",
      ">\n",
      "> It's that time of the year again... make it easy on\n",
      "> yourself.\n",
      "> Click here to get TurboTax!\n",
      ">\n",
      "http://service.bfast.com/bfast/click?bfmid=3D32881117&siteid=3D36901868&bfp=\n",
      "age=3Dtur\n",
      "botax\n",
      ">\n",
      "> Need money for college?\n",
      "> FastWeb is the largest FREE scholarship search on\n",
      "> the Internet, with accurate, updated information on\n",
      "> more than 600,000 scholarships, grants, and\n",
      "> fellowships.\n",
      "> http://click.emazing.com/ads/fastweb/text2.html\n",
      ">\n",
      "> Get satellite television in only one step.\n",
      "> Click here for more information on FREE satellite\n",
      "> TV!\n",
      ">\n",
      "http://www.commission-junction.com/track/track.dll?AID=3D1203660&PID=3D5521=\n",
      "59&URL=3D\n",
      "http%3A%2F%2Fwww%2EskyboxTV%2Ecom\n",
      ">\n",
      "> Send Fresh Flowers and Gifts from FTD.COM Today!\n",
      "> Don't forget, Valentine's Day is Wednesday, February\n",
      "> 14th. BE A HERO AT FTD.com.\n",
      "> http://click.emazing.com/ads/ftd/flowers.html\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      "> Another new year...\n",
      "> Another reason to send an EMAZING Greeting Card!\n",
      "> 2001 is\n",
      "> going to be full of birthdays, anniversaries,\n",
      "> holidays and\n",
      "> everyday occasions. Don't miss out--send a FREE and\n",
      "> fun\n",
      "> EMAZING Greeting Card today!\n",
      "> http://greetings.emazing.com/\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Quote of the Day\n",
      ">\n",
      "> Some cause happiness wherever they go; others,\n",
      "> whenever they go.\n",
      "> - Oscar Wilde\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> EMAZING SUBSCRIBER SERVICES\n",
      ">\n",
      "> EMAZING Home\n",
      "> http://www.emazing.com/\n",
      ">\n",
      "> Advice and Living\n",
      "> http://www.emazing.com/advice/\n",
      ">\n",
      "> Brain Candy\n",
      "> http://www.emazing.com/brain/\n",
      ">\n",
      "> Business and Money\n",
      "> http://www.emazing.com/business/\n",
      ">\n",
      "> Comics\n",
      "> http://www.emazing.com/comics/\n",
      ">\n",
      "> Daybreakers\n",
      "> http://www.emazing.com/daybreakers/\n",
      ">\n",
      "> Entertainment\n",
      "> http://www.emazing.com/entertainment/\n",
      ">\n",
      "> Espa=0Fol\n",
      "> http://espanol.emazing.com/\n",
      ">\n",
      "> Games\n",
      "> http://games.emazing.com/\n",
      ">\n",
      "> Good Thoughts\n",
      "> http://www.emazing.com/goodthoughts/\n",
      ">\n",
      "> Grapevine\n",
      "> http://grapevine.emazing.com/\n",
      ">\n",
      "> Greetings\n",
      "> http://greetings.emazing.com/\n",
      ">\n",
      "> Health\n",
      "> http://www.emazing.com/health/\n",
      ">\n",
      "> Hobbies\n",
      "> http://www.emazing.com/hobbies/\n",
      ">\n",
      "> Horoscopes\n",
      "> http://www.emazing.com/horoscopes/\n",
      ">\n",
      "> Household Hints\n",
      "> http://www.emazing.com/household/\n",
      ">\n",
      "> Kitchen\n",
      "> http://www.emazing.com/kitchen/\n",
      ">\n",
      "> Music\n",
      "> http://www.emazing.com/music/\n",
      ">\n",
      "> Sports\n",
      "> http://www.emazing.com/sports/\n",
      ">\n",
      "> Style\n",
      "> http://www.emazing.com/style/\n",
      ">\n",
      "> Technology\n",
      "> http://www.emazing.com/technology/\n",
      ">\n",
      "> Travel\n",
      "> http://www.emazing.com/travel/\n",
      ">\n",
      "> Wireless\n",
      "> http://www.emazing.com/wireless/\n",
      ">\n",
      "> Unsubscribe\n",
      "> http://www.emazing.com/unsub/recipe\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> Advertise\n",
      "> http://www.emazing.com/info/advertise.htm\n",
      ">\n",
      "> Archives\n",
      "> http://www.emazing.com/archives/\n",
      ">\n",
      "> Company information\n",
      "> http://www.emazing.com/info/\n",
      ">\n",
      "> Edit Your Profile\n",
      "> http://www.emazing.com/profile/\n",
      ">\n",
      "> Feedback\n",
      "> Got comments? Suggestions? Complaints?\n",
      "> Send us an email to feedback@emazing.com\n",
      ">\n",
      "> Help\n",
      "> http://www.emazing.com/info/help.htm\n",
      ">\n",
      "> Jobs\n",
      "> http://www.emazing.com/info/jobs.htm\n",
      ">\n",
      "> Privacy Policy\n",
      "> http://www.emazing.com/info/privacy.htm\n",
      ">\n",
      "> Switch To HTML Format\n",
      "> http://www.emazing.com/switchtohtml.htm\n",
      ">\n",
      "> Terms & Conditions\n",
      "> http://www.emazing.com/info/terms.htm\n",
      ">\n",
      "> EMAZING(tm) is a service of EMAZING, INC.\n",
      "> Copyright 2001. All Rights Reserved.\n",
      ">\n",
      ">\n",
      "---------------------------------------------------------\n",
      ">\n",
      "> EMAZING - Whatever you're into, we email it to you.\n",
      "> FREE!\n",
      "> http://www.emazing.com\n",
      ">\n",
      ">\n",
      ">\n",
      "\n",
      "\n",
      "__________________________________________________\n",
      "Do You Yahoo!?\n",
      "Yahoo! Photos - Share your holiday photos online!\n",
      "http://photos.yahoo.com/\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['content'].str.contains('HTML')]['content'].iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "16b618b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html is better but either one works\n",
      "\n",
      "\n",
      "To: Eric Bass/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Re: Fantast Basketball DRAFT TODAY @ 10:30  \n",
      "\n",
      "Rude Boys are in da house... Can't remember... Do we choose Java or HTML...\n",
      "\n",
      "\n",
      "   \n",
      "\tEnron North America Corp.\n",
      "\t\n",
      "\tFrom:  Eric Bass                           10/31/2000 08:25 AM\n",
      "\t\n",
      "\n",
      "To: David Baumbach/HOU/ECT@ECT, Phillip M Love/HOU/ECT@ECT, Bryan \n",
      "Hull/HOU/ECT@ECT, Harry Bucalo/HOU/ECT@ECT, O'Neal D Winfree/HOU/ECT@ECT, \n",
      "Michael Walters/HOU/ECT@ECT, Matthew Lenhart/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Fantast Basketball DRAFT TODAY @ 10:30\n",
      "\n",
      "Tell me if you are in on the draft at 10:30.  To draft you need to go to the \n",
      "website and click on draft - go to the draft room.\n",
      "---------------------- Forwarded by Eric Bass/HOU/ECT on 10/31/2000 08:23 AM \n",
      "---------------------------\n",
      "   \n",
      "\tEnron North America Corp.\n",
      "\t\n",
      "\tFrom:  Eric Bass                           10/31/2000 08:21 AM\n",
      "\t\n",
      "\n",
      "To: David Baumbach/HOU/ECT@ECT, Phillip M Love/HOU/ECT@ECT, Bryan \n",
      "Hull/HOU/ECT@ECT, Harry Bucalo/HOU/ECT@ECT, O'Neal D Winfree/HOU/ECT@ECT, \n",
      "Michael Walters/HOU/ECT@ECT, Matthew Lenhart/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Fantast Basketball DRAFT TODAY @ 10:30\n",
      "\n",
      "Just as clarification - we start 6 players and have weekly scoring in haed to \n",
      "head matchups.  You can chage your lineup before each day's worth of action.  \n",
      "Your sixth man can be any position.  The draft order will be determind at \n",
      "10:30.\n",
      "---------------------- Forwarded by Eric Bass/HOU/ECT on 10/31/2000 08:19 AM \n",
      "---------------------------\n",
      "   \n",
      "\tEnron North America Corp.\n",
      "\t\n",
      "\tFrom:  Eric Bass                           10/31/2000 08:01 AM\n",
      "\t\n",
      "\n",
      "To: David Baumbach/HOU/ECT@ECT, Phillip M Love/HOU/ECT@ECT, Bryan \n",
      "Hull/HOU/ECT@ECT, Harry Bucalo/HOU/ECT@ECT, O'Neal D Winfree/HOU/ECT@ECT, \n",
      "Michael Walters/HOU/ECT@ECT, Matthew Lenhart/HOU/ECT@ECT\n",
      "cc:  \n",
      "Subject: Fantast Basketball DRAFT TODAY @ 10:30\n",
      "\n",
      "Since the NBA season begins tonight, we need to have the fantasy basketball \n",
      "draft today @ 10:30 AM. Tell me if this is ok.  If you do not remember the \n",
      "rules, go to the link below and scroll down to the bottom of the page to the \n",
      "rules link.\n",
      "\n",
      "The Academy will roll once again!\n",
      "\n",
      "And by the way plove, it is last day for everyone.  You can take 1 minute out \n",
      "of every half hour today to make a pick.  Everyone else has agreed.  Besides \n",
      "it won't be as much fun without my whipping boy again.\n",
      "\n",
      "basketball.commissioner.com/efba\n",
      " \n",
      "I never heard anything back from Walker's Westies so The Rounders are taking \n",
      "his spot.\n",
      "\n",
      "-Eric\n"
     ]
    }
   ],
   "source": [
    "print(re_emails[re_emails['content'].str.contains('HTML')]['content'].iloc[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b9300691",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "1       Traveling to have a business meeting takes the...      139    False  \n",
       "2                          test successful.  way to go!!!        5    False  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False  \n",
       "7                        any morning between 10 and 11:30        6    False  \n",
       "...                                                   ...      ...      ...  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87    False  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75    False  \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False  \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False  \n",
       "\n",
       "[103469 rows x 5 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "93550653",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/103469 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      " 75%|█████████████████████████▌        | 77785/103469 [00:11<00:03, 6894.67it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|█████████████████████████████████| 103469/103469 [00:15<00:00, 6730.49it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \n",
       "1       Traveling to have a business meeting takes the...      139    False  \n",
       "2                          test successful.  way to go!!!        5    False  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False  \n",
       "7                        any morning between 10 and 11:30        6    False  \n",
       "...                                                   ...      ...      ...  \n",
       "495073  yeah I understand\\n----- Original Message ----...       87    False  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75    False  \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False  \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False  \n",
       "\n",
       "[103469 rows x 5 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract textual content from Html\n",
    "\n",
    "re_emails['content'] = re_emails['content'].progress_apply(lambda x: extract_html_content(x))\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "00cf547b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████| 103469/103469 [00:03<00:00, 27716.88it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "      <th>clean_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>test successful.  way to go!!!\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>any morning between 10 and 11:30\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "      <td>yeah I understand\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "      <td>i will attend\\n\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "      <td>sure\\n\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \\\n",
       "1       Traveling to have a business meeting takes the...      139    False   \n",
       "2                          test successful.  way to go!!!        5    False   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False   \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False   \n",
       "7                        any morning between 10 and 11:30        6    False   \n",
       "...                                                   ...      ...      ...   \n",
       "495073  yeah I understand\\n----- Original Message ----...       87    False   \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False   \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75    False   \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False   \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False   \n",
       "\n",
       "                                            clean_content  \n",
       "1       Traveling to have a business meeting takes the...  \n",
       "2                        test successful.  way to go!!!\\n  \n",
       "4                     Let's shoot for Tuesday at 11:45.\\n  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...  \n",
       "7                      any morning between 10 and 11:30\\n  \n",
       "...                                                   ...  \n",
       "495073                                yeah I understand\\n  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...  \n",
       "495329                                  i will attend\\n\\n  \n",
       "495405                                           sure\\n\\n  \n",
       "\n",
       "[103469 rows x 6 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract the top message splitting content by different patterns\n",
    "\n",
    "re_emails[\"clean_content\"] = re_emails[\"content\"].progress_apply(\n",
    "    lambda x: extract_content(\n",
    "        x,\n",
    "        stop_regexes=[\n",
    "            re.compile('\\s*Original Message'),\n",
    "            re.compile('\\s*Forwarded by'),\n",
    "            re.compile('^\\s?>?(From|To):\\s?', re.I),\n",
    "            re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}', re.I),\n",
    "            re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2} (AM|PM)', re.I),\n",
    "            re.compile('\\son \\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}:\\d{2} (AM|PM)$', re.I)\n",
    "        ]\n",
    "    )\n",
    ")\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "da54000b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/103469 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      " 74%|████████████████████████▌        | 77022/103469 [00:04<00:01, 20079.74it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|████████████████████████████████| 103469/103469 [00:05<00:00, 17493.01it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>hasHtml</th>\n",
       "      <th>clean_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>False</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>test successful.  way to go!!!\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;30965995.1075863688265.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>Greg,\\n\\n How about either next Tuesday or Thu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>&lt;17189699.1075863688308.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: PRC review - phone calls</td>\n",
       "      <td>any morning between 10 and 11:30</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>any morning between 10 and 11:30\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495073</th>\n",
       "      <td>&lt;3806936.1075842022389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Dec 19 marks</td>\n",
       "      <td>yeah I understand\\n----- Original Message ----...</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "      <td>yeah I understand\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495128</th>\n",
       "      <td>&lt;7342891.1075842025392.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Market Risk Meeting Today - John Zufferli</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "      <td>190</td>\n",
       "      <td>False</td>\n",
       "      <td>Hi.\\n\\n     The Market Risk Meeting has been s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495170</th>\n",
       "      <td>&lt;12973134.1075842026566.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Trades</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495329</th>\n",
       "      <td>&lt;19944775.1075842032737.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: REMINDER - Enron Presentation Re: Financia...</td>\n",
       "      <td>i will attend\\n\\n-----Original Message-----\\nF...</td>\n",
       "      <td>137</td>\n",
       "      <td>False</td>\n",
       "      <td>i will attend\\n\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495405</th>\n",
       "      <td>&lt;27443714.1075842034488.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:  Enron Presentation Re: Financial Planning</td>\n",
       "      <td>sure\\n\\n -----Original Message-----\\nFrom: \\tC...</td>\n",
       "      <td>413</td>\n",
       "      <td>False</td>\n",
       "      <td>sure\\n\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>103469 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "5       <30965995.1075863688265.JavaMail.evans@thyme>   \n",
       "7       <17189699.1075863688308.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495073   <3806936.1075842022389.JavaMail.evans@thyme>   \n",
       "495128   <7342891.1075842025392.JavaMail.evans@thyme>   \n",
       "495170  <12973134.1075842026566.JavaMail.evans@thyme>   \n",
       "495329  <19944775.1075842032737.JavaMail.evans@thyme>   \n",
       "495405  <27443714.1075842034488.JavaMail.evans@thyme>   \n",
       "\n",
       "                                                  subject  \\\n",
       "1                                                     Re:   \n",
       "2                                                Re: test   \n",
       "4                                               Re: Hello   \n",
       "5                                               Re: Hello   \n",
       "7                            Re: PRC review - phone calls   \n",
       "...                                                   ...   \n",
       "495073                                   Re: Dec 19 marks   \n",
       "495128      Re: Market Risk Meeting Today - John Zufferli   \n",
       "495170                                         Re: Trades   \n",
       "495329  RE: REMINDER - Enron Presentation Re: Financia...   \n",
       "495405     RE:  Enron Presentation Re: Financial Planning   \n",
       "\n",
       "                                                  content  n_words  hasHtml  \\\n",
       "1       Traveling to have a business meeting takes the...      139    False   \n",
       "2                          test successful.  way to go!!!        5    False   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6    False   \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...        9    False   \n",
       "7                        any morning between 10 and 11:30        6    False   \n",
       "...                                                   ...      ...      ...   \n",
       "495073  yeah I understand\\n----- Original Message ----...       87    False   \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...      190    False   \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...       75    False   \n",
       "495329  i will attend\\n\\n-----Original Message-----\\nF...      137    False   \n",
       "495405  sure\\n\\n -----Original Message-----\\nFrom: \\tC...      413    False   \n",
       "\n",
       "                                            clean_content  \n",
       "1       Traveling to have a business meeting takes the...  \n",
       "2                        test successful.  way to go!!!\\n  \n",
       "4                     Let's shoot for Tuesday at 11:45.\\n  \n",
       "5       Greg,\\n\\n How about either next Tuesday or Thu...  \n",
       "7                      any morning between 10 and 11:30\\n  \n",
       "...                                                   ...  \n",
       "495073                                yeah I understand\\n  \n",
       "495128  Hi.\\n\\n     The Market Risk Meeting has been s...  \n",
       "495170  Friday Jan 18 ,\\n\\n\\nbom traded 29\\nfeb traded...  \n",
       "495329                                  i will attend\\n\\n  \n",
       "495405                                           sure\\n\\n  \n",
       "\n",
       "[103469 rows x 6 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check again how many emails contains Html tags after extracting content\n",
    "\n",
    "re_emails['hasHtml'] = re_emails['content'].progress_apply(lambda x: hasHtml(x))\n",
    "re_emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "44ecd895",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    103469\n",
       "Name: hasHtml, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re_emails['hasHtml'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "585c88bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \n",
       "0                                    Here is our forecast        4  \n",
       "1       Traveling to have a business meeting takes the...      139  \n",
       "2                          test successful.  way to go!!!        5  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6  \n",
       "...                                                   ...      ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48  \n",
       "495543  Some of my position is with the Alberta Term b...       23  \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27  \n",
       "495546  i think the YMCA has a class that is for peopl...      575  \n",
       "\n",
       "[495547 rows x 4 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e25f6f43",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/495547 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "  0%|                                      | 68/495547 [00:00<12:09, 679.57it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|█████████████████████████████████| 495547/495547 [01:30<00:00, 5466.66it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \n",
       "0                                    Here is our forecast        4  \n",
       "1       Traveling to have a business meeting takes the...      139  \n",
       "2                          test successful.  way to go!!!        5  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34  \n",
       "4                       Let's shoot for Tuesday at 11:45.        6  \n",
       "...                                                   ...      ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48  \n",
       "495543  Some of my position is with the Alberta Term b...       23  \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27  \n",
       "495546  i think the YMCA has a class that is for peopl...      575  \n",
       "\n",
       "[495547 rows x 4 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We'll apply the same steps for the entire dataset\n",
    "\n",
    "# 1. Extract textual content from HTML\n",
    "# 2. Split the content by regex patterns to extract the top message\n",
    "# 3. Apply textual cleansing e.g. remove beggining of text starting with >, >>, remove remaining HTML tags, deal with ==02,\n",
    "# 4. Remove duplicate emails\n",
    "\n",
    "# Extract textual content from Html\n",
    "df['content'] = df['content'].progress_apply(lambda x: extract_html_content(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "21c19b58",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████| 495547/495547 [00:46<00:00, 10595.57it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2\\n\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  \n",
       "0                                  Here is our forecast\\n  \n",
       "1       Traveling to have a business meeting takes the...  \n",
       "2                        test successful.  way to go!!!\\n  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...  \n",
       "4                     Let's shoot for Tuesday at 11:45.\\n  \n",
       "...                                                   ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...  \n",
       "495543  Some of my position is with the Alberta Term b...  \n",
       "495544                                              2\\n\\n  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...  \n",
       "495546  i think the YMCA has a class that is for peopl...  \n",
       "\n",
       "[495547 rows x 5 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract the top message splitting content by different patterns\n",
    "\n",
    "df[\"clean_content\"] = df[\"content\"].progress_apply(\n",
    "    lambda x: extract_content(\n",
    "        x,\n",
    "        stop_regexes=[\n",
    "            re.compile('\\s*Original Message'),\n",
    "            re.compile('\\s*Forwarded by'),\n",
    "            re.compile('^\\s?>?(From|To):\\s?', re.I),\n",
    "            re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}', re.I),\n",
    "            re.compile('\\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2} (AM|PM)', re.I),\n",
    "            re.compile('\\son \\d{2}\\/\\d{2}\\/\\d{2,4} \\d{2}:\\d{2}:\\d{2} (AM|PM)$', re.I)\n",
    "        ]\n",
    "    )\n",
    ")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "4519eb87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████| 495547/495547 [00:42<00:00, 11716.75it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  \n",
       "0                                    Here is our forecast  \n",
       "1       Traveling to have a business meeting takes the...  \n",
       "2                          test successful.  way to go!!!  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...  \n",
       "4                       Let's shoot for Tuesday at 11:45.  \n",
       "...                                                   ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...  \n",
       "495543  Some of my position is with the Alberta Term b...  \n",
       "495544                                                  2  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...  \n",
       "495546  i think the YMCA has a class that is for peopl...  \n",
       "\n",
       "[495547 rows x 5 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Apply textual cleansing\n",
    "\n",
    "df['clean_content'] = df['clean_content'].progress_apply(lambda x: clean_text(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "f0be1a33",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████| 495547/495547 [00:00<00:00, 1409581.72it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  \n",
       "0                                    Here is our forecast  \n",
       "1       Traveling to have a business meeting takes the...  \n",
       "2                          test successful.  way to go!!!  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...  \n",
       "4                       Let's shoot for Tuesday at 11:45.  \n",
       "...                                                   ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...  \n",
       "495543  Some of my position is with the Alberta Term b...  \n",
       "495544                                                  2  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...  \n",
       "495546  i think the YMCA has a class that is for peopl...  \n",
       "\n",
       "[495547 rows x 5 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove newlines found at the beginning and ending text\n",
    "\n",
    "df['clean_content'] = df['clean_content'].progress_apply(lambda x: x.strip())\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "0a11c106",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                | 0/495547 [00:00<?, ?it/s]/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  warnings.warn(\n",
      "/Users/daciantamasan/Desktop/anaconda3/lib/python3.10/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  warnings.warn(\n",
      "100%|████████████████████████████████| 495547/495547 [00:24<00:00, 20533.80it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>hasHtml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  hasHtml  \n",
       "0                                    Here is our forecast    False  \n",
       "1       Traveling to have a business meeting takes the...    False  \n",
       "2                          test successful.  way to go!!!    False  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...    False  \n",
       "4                       Let's shoot for Tuesday at 11:45.    False  \n",
       "...                                                   ...      ...  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...    False  \n",
       "495543  Some of my position is with the Alberta Term b...    False  \n",
       "495544                                                  2    False  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...    False  \n",
       "495546  i think the YMCA has a class that is for peopl...    False  \n",
       "\n",
       "[495547 rows x 6 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check how many emails contains Html tags after extracting and cleaning the text\n",
    "\n",
    "df['hasHtml'] = df['clean_content'].progress_apply(lambda x: hasHtml(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e46f81f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    495547\n",
       "Name: hasHtml, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['hasHtml'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "adbfff2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████| 495547/495547 [00:02<00:00, 170596.25it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>hasHtml</th>\n",
       "      <th>n_words_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>False</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>False</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>False</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>False</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>False</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>False</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495547 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  hasHtml  \\\n",
       "0                                    Here is our forecast    False   \n",
       "1       Traveling to have a business meeting takes the...    False   \n",
       "2                          test successful.  way to go!!!    False   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...    False   \n",
       "4                       Let's shoot for Tuesday at 11:45.    False   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...    False   \n",
       "495543  Some of my position is with the Alberta Term b...    False   \n",
       "495544                                                  2    False   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...    False   \n",
       "495546  i think the YMCA has a class that is for peopl...    False   \n",
       "\n",
       "        n_words_clean  \n",
       "0                   4  \n",
       "1                 139  \n",
       "2                   5  \n",
       "3                  34  \n",
       "4                   6  \n",
       "...               ...  \n",
       "495542             48  \n",
       "495543             23  \n",
       "495544              1  \n",
       "495545             27  \n",
       "495546             19  \n",
       "\n",
       "[495547 rows x 7 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Determine the number of words in clean text\n",
    "\n",
    "df['n_words_clean'] = df['clean_content'].progress_apply(lambda x: len(x.split()))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "aa40280e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    495547.000000\n",
       "mean        119.735767\n",
       "std         607.733803\n",
       "min           0.000000\n",
       "25%          10.000000\n",
       "50%          32.000000\n",
       "75%          89.000000\n",
       "max       59686.000000\n",
       "Name: n_words_clean, dtype: float64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['n_words_clean'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "7c7e1b7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>hasHtml</th>\n",
       "      <th>n_words_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>False</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>False</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>False</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>False</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>False</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>False</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>451536 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495544           RE: CONFIDENTIAL   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  hasHtml  \\\n",
       "0                                    Here is our forecast    False   \n",
       "1       Traveling to have a business meeting takes the...    False   \n",
       "2                          test successful.  way to go!!!    False   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...    False   \n",
       "4                       Let's shoot for Tuesday at 11:45.    False   \n",
       "...                                                   ...      ...   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...    False   \n",
       "495543  Some of my position is with the Alberta Term b...    False   \n",
       "495544                                                  2    False   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...    False   \n",
       "495546  i think the YMCA has a class that is for peopl...    False   \n",
       "\n",
       "        n_words_clean  \n",
       "0                   4  \n",
       "1                 139  \n",
       "2                   5  \n",
       "3                  34  \n",
       "4                   6  \n",
       "...               ...  \n",
       "495542             48  \n",
       "495543             23  \n",
       "495544              1  \n",
       "495545             27  \n",
       "495546             19  \n",
       "\n",
       "[451536 rows x 7 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter out emails with no words\n",
    "\n",
    "df = df[df['n_words_clean'] != 0]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "1279c9e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    451536.000000\n",
       "mean        131.406355\n",
       "std         635.457547\n",
       "min           1.000000\n",
       "25%          15.000000\n",
       "50%          38.000000\n",
       "75%         100.000000\n",
       "max       59686.000000\n",
       "Name: n_words_clean, dtype: float64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['n_words_clean'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "af40b216",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>hasHtml</th>\n",
       "      <th>n_words_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>&lt;4070589.1075855723617.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>&lt;30467968.1075855723641.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: 2- SURVEY/INFORMATION EMAIL 5-14- 01</td>\n",
       "      <td>Outlook Migration Team@ENRON\\n05/11/2001 01:49...</td>\n",
       "      <td>502</td>\n",
       "      <td>Outlook Migration Team@ENRON</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340</th>\n",
       "      <td>&lt;30542109.1075855379282.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: Presentation to Trading Track A&amp;A</td>\n",
       "      <td>Ina, \\n\\nCan you please forward the presentati...</td>\n",
       "      <td>109</td>\n",
       "      <td>Ina, \\n\\nCan you please forward the presentati...</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>&lt;5860470.1075855667730.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Dave, \\n\\n Here are the names of the west desk...</td>\n",
       "      <td>19</td>\n",
       "      <td>Dave, \\n\\n Here are the names of the west desk...</td>\n",
       "      <td>False</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>&lt;14670081.1075855667751.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: 2001 Margin Plan</td>\n",
       "      <td>Paula,\\n\\n 35 million is fine\\n\\nPhillip</td>\n",
       "      <td>6</td>\n",
       "      <td>Paula,\\n\\n 35 million is fine\\n\\nPhillip</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495485</th>\n",
       "      <td>&lt;5407571.1075842028586.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>FW: 1996 Porsche 993 C4S</td>\n",
       "      <td>i lifted it last night for 82,000\\n -----Origi...</td>\n",
       "      <td>84</td>\n",
       "      <td>i lifted it last night for 82,000</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495503</th>\n",
       "      <td>&lt;3836124.1075842027707.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>1998 Silver C4S</td>\n",
       "      <td>Check out www.segalmotorcar.com</td>\n",
       "      <td>3</td>\n",
       "      <td>Check out www.segalmotorcar.com</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495519</th>\n",
       "      <td>&lt;16498082.1075842029389.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: W/C Deficiency Rider</td>\n",
       "      <td>yes\\n\\n -----Original Message-----\\nFrom: \\tDu...</td>\n",
       "      <td>90</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495534</th>\n",
       "      <td>&lt;11279427.1075842029748.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>FW: Strictly Confidential</td>\n",
       "      <td>Here is the info for the commercial staff\\n\\nJ...</td>\n",
       "      <td>41</td>\n",
       "      <td>Here is the info for the commercial staff\\n\\nJ...</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495544</th>\n",
       "      <td>&lt;28979867.1075842029988.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: CONFIDENTIAL</td>\n",
       "      <td>2\\n\\n -----Original Message-----\\nFrom: \\tDouc...</td>\n",
       "      <td>54</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>239639 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "276      <4070589.1075855723617.JavaMail.evans@thyme>   \n",
       "277     <30467968.1075855723641.JavaMail.evans@thyme>   \n",
       "340     <30542109.1075855379282.JavaMail.evans@thyme>   \n",
       "604      <5860470.1075855667730.JavaMail.evans@thyme>   \n",
       "605     <14670081.1075855667751.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495485   <5407571.1075842028586.JavaMail.evans@thyme>   \n",
       "495503   <3836124.1075842027707.JavaMail.evans@thyme>   \n",
       "495519  <16498082.1075842029389.JavaMail.evans@thyme>   \n",
       "495534  <11279427.1075842029748.JavaMail.evans@thyme>   \n",
       "495544  <28979867.1075842029988.JavaMail.evans@thyme>   \n",
       "\n",
       "                                         subject  \\\n",
       "276                                                \n",
       "277     Re: 2- SURVEY/INFORMATION EMAIL 5-14- 01   \n",
       "340        RE: Presentation to Trading Track A&A   \n",
       "604                                                \n",
       "605                         Re: 2001 Margin Plan   \n",
       "...                                          ...   \n",
       "495485                  FW: 1996 Porsche 993 C4S   \n",
       "495503                           1998 Silver C4S   \n",
       "495519                  RE: W/C Deficiency Rider   \n",
       "495534                 FW: Strictly Confidential   \n",
       "495544                          RE: CONFIDENTIAL   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "276                                  Here is our forecast        4   \n",
       "277     Outlook Migration Team@ENRON\\n05/11/2001 01:49...      502   \n",
       "340     Ina, \\n\\nCan you please forward the presentati...      109   \n",
       "604     Dave, \\n\\n Here are the names of the west desk...       19   \n",
       "605              Paula,\\n\\n 35 million is fine\\n\\nPhillip        6   \n",
       "...                                                   ...      ...   \n",
       "495485  i lifted it last night for 82,000\\n -----Origi...       84   \n",
       "495503                    Check out www.segalmotorcar.com        3   \n",
       "495519  yes\\n\\n -----Original Message-----\\nFrom: \\tDu...       90   \n",
       "495534  Here is the info for the commercial staff\\n\\nJ...       41   \n",
       "495544  2\\n\\n -----Original Message-----\\nFrom: \\tDouc...       54   \n",
       "\n",
       "                                            clean_content  hasHtml  \\\n",
       "276                                  Here is our forecast    False   \n",
       "277                          Outlook Migration Team@ENRON    False   \n",
       "340     Ina, \\n\\nCan you please forward the presentati...    False   \n",
       "604     Dave, \\n\\n Here are the names of the west desk...    False   \n",
       "605              Paula,\\n\\n 35 million is fine\\n\\nPhillip    False   \n",
       "...                                                   ...      ...   \n",
       "495485                  i lifted it last night for 82,000    False   \n",
       "495503                    Check out www.segalmotorcar.com    False   \n",
       "495519                                                yes    False   \n",
       "495534  Here is the info for the commercial staff\\n\\nJ...    False   \n",
       "495544                                                  2    False   \n",
       "\n",
       "        n_words_clean  \n",
       "276                 4  \n",
       "277                 3  \n",
       "340                10  \n",
       "604                19  \n",
       "605                 6  \n",
       "...               ...  \n",
       "495485              7  \n",
       "495503              3  \n",
       "495519              1  \n",
       "495534             10  \n",
       "495544              1  \n",
       "\n",
       "[239639 rows x 7 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check how many duplicate emails are\n",
    "\n",
    "dup_df = df[df.duplicated(subset=['clean_content'])]\n",
    "dup_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "41adc217",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>subject</th>\n",
       "      <th>content</th>\n",
       "      <th>n_words</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>hasHtml</th>\n",
       "      <th>n_words_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re:</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>False</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td></td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>False</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495541</th>\n",
       "      <td>&lt;20245656.1075842029914.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE:</td>\n",
       "      <td>very busy, jess is ok just scared\\n\\n -----Ori...</td>\n",
       "      <td>122</td>\n",
       "      <td>very busy, jess is ok just scared</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Trade with John Lavorato</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>False</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Gas Hedges</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>False</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Calgary Analyst/Associate</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>False</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>RE: ali's essays</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>575</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>False</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>211897 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495541  <20245656.1075842029914.JavaMail.evans@thyme>   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                          subject  \\\n",
       "0                                   \n",
       "1                             Re:   \n",
       "2                        Re: test   \n",
       "3                                   \n",
       "4                       Re: Hello   \n",
       "...                           ...   \n",
       "495541                        RE:   \n",
       "495542   Trade with John Lavorato   \n",
       "495543                 Gas Hedges   \n",
       "495545  Calgary Analyst/Associate   \n",
       "495546           RE: ali's essays   \n",
       "\n",
       "                                                  content  n_words  \\\n",
       "0                                    Here is our forecast        4   \n",
       "1       Traveling to have a business meeting takes the...      139   \n",
       "2                          test successful.  way to go!!!        5   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...       34   \n",
       "4                       Let's shoot for Tuesday at 11:45.        6   \n",
       "...                                                   ...      ...   \n",
       "495541  very busy, jess is ok just scared\\n\\n -----Ori...      122   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...       48   \n",
       "495543  Some of my position is with the Alberta Term b...       23   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...       27   \n",
       "495546  i think the YMCA has a class that is for peopl...      575   \n",
       "\n",
       "                                            clean_content  hasHtml  \\\n",
       "0                                    Here is our forecast    False   \n",
       "1       Traveling to have a business meeting takes the...    False   \n",
       "2                          test successful.  way to go!!!    False   \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...    False   \n",
       "4                       Let's shoot for Tuesday at 11:45.    False   \n",
       "...                                                   ...      ...   \n",
       "495541                  very busy, jess is ok just scared    False   \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...    False   \n",
       "495543  Some of my position is with the Alberta Term b...    False   \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...    False   \n",
       "495546  i think the YMCA has a class that is for peopl...    False   \n",
       "\n",
       "        n_words_clean  \n",
       "0                   4  \n",
       "1                 139  \n",
       "2                   5  \n",
       "3                  34  \n",
       "4                   6  \n",
       "...               ...  \n",
       "495541              7  \n",
       "495542             48  \n",
       "495543             23  \n",
       "495545             27  \n",
       "495546             19  \n",
       "\n",
       "[211897 rows x 7 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop duplicates\n",
    "\n",
    "df = df[~df.duplicated(subset=['clean_content'])]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "993b89e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message_id</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>n_words_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;18782981.1075855378110.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Here is our forecast</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;15464986.1075855378456.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Traveling to have a business meeting takes the...</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;24216240.1075855687451.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;13505866.1075863688222.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Randy,\\n\\n Can you send me a schedule of the s...</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;30922949.1075863688243.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Let's shoot for Tuesday at 11:45.</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495541</th>\n",
       "      <td>&lt;20245656.1075842029914.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>very busy, jess is ok just scared</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495542</th>\n",
       "      <td>&lt;26807948.1075842029936.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>This is a trade with OIL-SPEC-HEDGE-NG (John L...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495543</th>\n",
       "      <td>&lt;25835861.1075842029959.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Some of my position is with the Alberta Term b...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495545</th>\n",
       "      <td>&lt;22052556.1075842030013.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495546</th>\n",
       "      <td>&lt;28618979.1075842030037.JavaMail.evans@thyme&gt;</td>\n",
       "      <td>i think the YMCA has a class that is for peopl...</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>211897 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           message_id  \\\n",
       "0       <18782981.1075855378110.JavaMail.evans@thyme>   \n",
       "1       <15464986.1075855378456.JavaMail.evans@thyme>   \n",
       "2       <24216240.1075855687451.JavaMail.evans@thyme>   \n",
       "3       <13505866.1075863688222.JavaMail.evans@thyme>   \n",
       "4       <30922949.1075863688243.JavaMail.evans@thyme>   \n",
       "...                                               ...   \n",
       "495541  <20245656.1075842029914.JavaMail.evans@thyme>   \n",
       "495542  <26807948.1075842029936.JavaMail.evans@thyme>   \n",
       "495543  <25835861.1075842029959.JavaMail.evans@thyme>   \n",
       "495545  <22052556.1075842030013.JavaMail.evans@thyme>   \n",
       "495546  <28618979.1075842030037.JavaMail.evans@thyme>   \n",
       "\n",
       "                                            clean_content  n_words_clean  \n",
       "0                                    Here is our forecast              4  \n",
       "1       Traveling to have a business meeting takes the...            139  \n",
       "2                          test successful.  way to go!!!              5  \n",
       "3       Randy,\\n\\n Can you send me a schedule of the s...             34  \n",
       "4                       Let's shoot for Tuesday at 11:45.              6  \n",
       "...                                                   ...            ...  \n",
       "495541                  very busy, jess is ok just scared              7  \n",
       "495542  This is a trade with OIL-SPEC-HEDGE-NG (John L...             48  \n",
       "495543  Some of my position is with the Alberta Term b...             23  \n",
       "495545  Analyst\\t\\t\\t\\t\\tRank\\n\\nStephane Brodeur\\t\\t\\...             27  \n",
       "495546  i think the YMCA has a class that is for peopl...             19  \n",
       "\n",
       "[211897 rows x 3 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Keep only relevant column names\n",
    "\n",
    "df = df[['message_id', 'clean_content', 'n_words_clean']]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "575452af",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"data/clean_emails.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}