Phase_02.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    "# Table of Contents\n",
    " <p><div class=\"lev1 toc-item\"><a href=\"#Setup\" data-toc-modified-id=\"Setup-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Setup</a></div><div class=\"lev1 toc-item\"><a href=\"#Semantics\" data-toc-modified-id=\"Semantics-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Semantics</a></div><div class=\"lev2 toc-item\"><a href=\"#Motivation\" data-toc-modified-id=\"Motivation-21\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Motivation</a></div><div class=\"lev2 toc-item\"><a href=\"#Term-Document\" data-toc-modified-id=\"Term-Document-22\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Term-Document</a></div><div class=\"lev3 toc-item\"><a href=\"#Bag-of-Words\" data-toc-modified-id=\"Bag-of-Words-221\"><span class=\"toc-item-num\">2.2.1&nbsp;&nbsp;</span>Bag-of-Words</a></div><div class=\"lev3 toc-item\"><a href=\"#TF-IDF\" data-toc-modified-id=\"TF-IDF-222\"><span class=\"toc-item-num\">2.2.2&nbsp;&nbsp;</span>TF-IDF</a></div><div class=\"lev2 toc-item\"><a href=\"#Term-Context\" data-toc-modified-id=\"Term-Context-23\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Term-Context</a></div><div class=\"lev1 toc-item\"><a href=\"#Word2Vec\" data-toc-modified-id=\"Word2Vec-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Word2Vec</a></div><div class=\"lev2 toc-item\"><a href=\"#Continuous-Bag-of-Words\" data-toc-modified-id=\"Continuous-Bag-of-Words-31\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Continuous Bag of Words</a></div><div class=\"lev2 toc-item\"><a href=\"#Skip-gram\" data-toc-modified-id=\"Skip-gram-32\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Skip-gram</a></div><div class=\"lev2 toc-item\"><a href=\"#Example\" data-toc-modified-id=\"Example-33\"><span class=\"toc-item-num\">3.3&nbsp;&nbsp;</span>Example</a></div><div class=\"lev1 toc-item\"><a href=\"#Doc2Vec\" data-toc-modified-id=\"Doc2Vec-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Doc2Vec</a></div><div class=\"lev2 toc-item\"><a href=\"#Doc2Vec,-the-most-powerful-extension-of-word2vec\" data-toc-modified-id=\"Doc2Vec,-the-most-powerful-extension-of-word2vec-41\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>Doc2Vec, the most powerful extension of word2vec</a></div><div class=\"lev2 toc-item\"><a href=\"#Distrubted-Memory-(DM)\" data-toc-modified-id=\"Distrubted-Memory-(DM)-42\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>Distrubted Memory (DM)</a></div><div class=\"lev2 toc-item\"><a href=\"#Distrubted-Bag-of-Words-(DBOW)\" data-toc-modified-id=\"Distrubted-Bag-of-Words-(DBOW)-43\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>Distrubted Bag of Words (DBOW)</a></div><div class=\"lev1 toc-item\"><a href=\"#Exercises\" data-toc-modified-id=\"Exercises-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Exercises</a></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup\n",
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook assumes you have done the setup required in Week 1.\n",
    "\n",
    "In this lecture we will be using Gensim and NLTK, two widely used Python Natural Language Processing libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset -f -s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pip_install(*packages):\n",
    "    \"\"\"\n",
    "    Install packages using pip\n",
    "    Alternatively just use command line\n",
    "    pip install package_name\n",
    "    \"\"\"\n",
    "    try:\n",
    "        import pip\n",
    "        for package in packages:\n",
    "            pip.main([\"install\", \"--upgrade\", package, \"--user\"])\n",
    "    except Exception as e:\n",
    "        print(\"Unable to install {} using pip.\".format(package))\n",
    "        print(\"Exception:\", e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gensim\n",
      "  Using cached gensim-3.2.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl\n",
      "Requirement already up-to-date: scipy>=0.18.1 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from gensim)\n",
      "Requirement already up-to-date: six>=1.5.0 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from gensim)\n",
      "Collecting smart-open>=1.2.1 (from gensim)\n",
      "Requirement already up-to-date: numpy>=1.11.3 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from gensim)\n",
      "Collecting boto>=2.32 (from smart-open>=1.2.1->gensim)\n",
      "  Using cached boto-2.48.0-py2.py3-none-any.whl\n",
      "Collecting boto3 (from smart-open>=1.2.1->gensim)\n",
      "  Using cached boto3-1.5.22-py2.py3-none-any.whl\n",
      "Collecting bz2file (from smart-open>=1.2.1->gensim)\n",
      "Requirement already up-to-date: requests in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from smart-open>=1.2.1->gensim)\n",
      "Collecting s3transfer<0.2.0,>=0.1.10 (from boto3->smart-open>=1.2.1->gensim)\n",
      "  Using cached s3transfer-0.1.12-py2.py3-none-any.whl\n",
      "Collecting jmespath<1.0.0,>=0.7.1 (from boto3->smart-open>=1.2.1->gensim)\n",
      "  Using cached jmespath-0.9.3-py2.py3-none-any.whl\n",
      "Collecting botocore<1.9.0,>=1.8.36 (from boto3->smart-open>=1.2.1->gensim)\n",
      "  Using cached botocore-1.8.36-py2.py3-none-any.whl\n",
      "Requirement already up-to-date: chardet<3.1.0,>=3.0.2 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already up-to-date: idna<2.7,>=2.5 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already up-to-date: urllib3<1.23,>=1.21.1 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already up-to-date: certifi>=2017.4.17 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Collecting docutils>=0.10 (from botocore<1.9.0,>=1.8.36->boto3->smart-open>=1.2.1->gensim)\n",
      "  Using cached docutils-0.14-py3-none-any.whl\n",
      "Requirement already up-to-date: python-dateutil<3.0.0,>=2.1 in /Users/lifesaver/miniconda3/lib/python3.6/site-packages (from botocore<1.9.0,>=1.8.36->boto3->smart-open>=1.2.1->gensim)\n",
      "Installing collected packages: boto, jmespath, docutils, botocore, s3transfer, boto3, bz2file, smart-open, gensim\n"
     ]
    }
   ],
   "source": [
    "pip_install('gensim', 'nltk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package gutenberg to\n",
      "[nltk_data]     /Users/lifesaver/nltk_data...\n",
      "[nltk_data]   Package gutenberg is already up-to-date!\n",
      "[nltk_data] Downloading package reuters to\n",
      "[nltk_data]     /Users/lifesaver/nltk_data...\n",
      "[nltk_data]   Package reuters is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('gutenberg')\n",
    "nltk.download('reuters')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "ROOTDIR = os.path.abspath(os.path.dirname('__file__'))\n",
    "DATADIR = os.path.join(ROOTDIR, 'data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Semantics\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Motivation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we want to be able to categorize text, we need to be able to generate features for articles, paragraphs, sentences and other bodies of text, based on the information they contain and what they represent. There are a number of ways to achieve this and we will go over 3 approaches."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Term-Document"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bag-of-Words\n",
    "\n",
    "One of the simplest ways to extract features from text is to just count how many times a word appears in a body of text. In this model, the order of words does not matter and only the number of occurrences of each unique term for each document is taken into account."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#Load movie reviews dataset\n",
    "df = pd.read_csv(os.path.join(DATADIR, 'movie_reviews.csv'), nrows=100000)\n",
    "texts = df.text.values #pd.Series -> np.ndarray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "# Transform each review string as a list of token strings. May take a few seconds\n",
    "tokenized = [nltk.word_tokenize(review) for review in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example review:\n",
      "   Raw: So ingenious in concept, design and execution that you could watch it on a postage stamp-sized screen and still be engulfed by its charm. \n",
      "\n",
      "   Tokenized: ['So', 'ingenious', 'in', 'concept', ',', 'design', 'and', 'execution', 'that', 'you', 'could', 'watch', 'it', 'on', 'a', 'postage', 'stamp-sized', 'screen', 'and', 'still', 'be', 'engulfed', 'by', 'its', 'charm', '.']\n"
     ]
    }
   ],
   "source": [
    "n = 10 #arbitrary pick\n",
    "print('Example review:\\n   Raw: {} \\n\\n   Tokenized: {}'.format(texts[n], [i for i in tokenized[n]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(tokenized_list):\n",
    "    import string\n",
    "    sw = nltk.corpus.stopwords.words('english')\n",
    "    new_list = [[token.lower() for token in tlist if token not in string.punctuation and token.lower() not in sw] for tlist in tokenized_list]\n",
    "    return new_list\n",
    "\n",
    "# Remove punctuations and stopwords\n",
    "cleaned = clean_text(tokenized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'pattern' package not found; tag filters are not available for English\n",
      "adding document #0 to Dictionary(0 unique tokens: [])\n",
      "adding document #10000 to Dictionary(19398 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #20000 to Dictionary(28254 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #30000 to Dictionary(34454 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #40000 to Dictionary(39899 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #50000 to Dictionary(45118 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #60000 to Dictionary(50200 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #70000 to Dictionary(54543 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #80000 to Dictionary(58569 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "adding document #90000 to Dictionary(62811 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...)\n",
      "built Dictionary(67734 unique tokens: ['...', 'entire', 'ever', 'filmgoers', 'generation']...) from 100000 documents (total 1149504 corpus positions)\n"
     ]
    }
   ],
   "source": [
    "from gensim import corpora\n",
    "\n",
    "# Create a dictionary from list of documents\n",
    "dictionary = corpora.Dictionary(cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example review featurized in Bag of Words :\n",
      " [('still', 1), ('charm', 1), ('concept', 1), ('could', 1), ('design', 1), ('engulfed', 1), ('execution', 1), ('ingenious', 1), ('postage', 1), ('screen', 1), ('stamp-sized', 1), ('watch', 1)]\n"
     ]
    }
   ],
   "source": [
    "# Create a Corpus based on BOW Format.\n",
    "corpus = [dictionary.doc2bow(text) for text in cleaned]\n",
    "print('Example review featurized in Bag of Words :\\n {}'.format([(dictionary[i[0]], i[1]) for i in corpus[n]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that when we use this model to featurize text:\n",
    "- The length of each feature vector will be the size of the vocabulary in the corpus\n",
    "- Thus each body of text will have a lot of zeroes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Term Frequency__: Number of occurrences of a word in a document  \n",
    "__Inverse Document Frequency__: Number of documents that contain a certain word scaled by a weight  \n",
    "__Term Frequency - Inverse Document Frequency__: (Number of ocurrences of word $w$ in text $T$) * $log$(Number of documents in a corpus/Number of documents containing word $w$)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check out the TF-IDF scores of the previous movie review we examined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "collecting document frequencies\n",
      "PROGRESS: processing document #0\n",
      "PROGRESS: processing document #10000\n",
      "PROGRESS: processing document #20000\n",
      "PROGRESS: processing document #30000\n",
      "PROGRESS: processing document #40000\n",
      "PROGRESS: processing document #50000\n",
      "PROGRESS: processing document #60000\n",
      "PROGRESS: processing document #70000\n",
      "PROGRESS: processing document #80000\n",
      "PROGRESS: processing document #90000\n",
      "calculating IDF weights for 100000 documents and 67733 features (1111018 matrix non-zeros)\n",
      "Example review featurized with TF-IDF scores : \n",
      "[('still', 0.152), ('charm', 0.206), ('concept', 0.244), ('could', 0.159), ('design', 0.25), ('engulfed', 0.43), ('execution', 0.259), ('ingenious', 0.293), ('postage', 0.414), ('screen', 0.176), ('stamp-sized', 0.458), ('watch', 0.191)]\n"
     ]
    }
   ],
   "source": [
    "from gensim import corpora, models\n",
    "\n",
    "#Create a TFIDF Model for the corpus\n",
    "tfidf = models.TfidfModel(corpus)\n",
    "\n",
    "print('Example review featurized with TF-IDF scores : \\n{}'.format([(dictionary[i[0]], round(i[1],3)) for i in tfidf[corpus[n]]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks much more like a feature vector that we can use for text categorization!  \n",
    "Note that in the TF-IDF model:\n",
    "- If a term frequently occurs in the corpus(i.e. stopwords, the term $like$), it is scaled to a lower score\n",
    "- Rarer terms will generally have higher scores. They tend to be more \"informative\" and descriptive.\n",
    "- A term that occurs frequently in a small number of documents within the corpus will have the highest scores."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](images/tfidf.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Term-Context"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The vast majority of NLP works regards as atomic symbols: king, queen, book, etc.\n",
    "\n",
    "In vector space terms, this vector has one $1 $ and a lot of zeros.   \n",
    "$king =  [1, 0, 0, 0, 0, 0, 0, 0, 0]$   \n",
    "$queen = [0, 1, 0, 0, 0, 0, 0, 0, 0]$   \n",
    "$book =  [0, 0, 1, 0, 0, 0, 0, 0, 0]$  \n",
    "\n",
    "It is called a \"one-hot\" encoding representation. It is a common way to represent categories in models. However, it is very sparse(as we saw from the BOW model); each row is mostly 0s.  \n",
    "You can get more value by representing a word by its neighorbors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instead of using entire documents, we can use small contexts to a term.\n",
    "- Paragraphs\n",
    "- Sentences\n",
    "- A window of a sequence of consecutive terms\n",
    "\n",
    "In this way, a word is defined over counts of context words. The assumption is that two words that appear in similar contexts are similar themselves."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But count-based models have disadvantages:\n",
    "- vector sizes become huge, equal to vocabulary size\n",
    "    - sparsity\n",
    "    - curse of dimensionality\n",
    "    - computationally expensive\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word2Vec is an unsupervised neural network model that maximizes similarity between contextual neighbors while minimizing similarity for unseen contexts.\n",
    "Initial vectors are generated randomly and converge as the models is trained on the corpus through a sliding window.\n",
    "Target Vector sizes are set at the beginning of the training process, so the vectors are dense and do not need dimensionality reduction techniques.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Continuous Bag of Words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](images/cbow.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training objective is to maiximize the probability of observing the correct target word $w_t$ given context words $w_{c1}, w_{c2}, ... w_{cj}$\n",
    "\n",
    " $$ C = -log p(w_t | w_{c1} ... w_{cj}) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The prediction vector is set as an average of all the context word vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Skip-gram"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](images/skip-gram.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training objective is to maiximize the probability of observing the correct context words $w_{ci}$ given target word $w_{t}$\n",
    "\n",
    " $$ C = -\\sum^{j}_{i=1}log p(w_{ci} | w_{t}) $$\n",
    " \n",
    "In this case, the prediction vector is the the target word vector."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's try training our own word embeddings and looking at what we can do with them."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word2Vec\n",
    "- `size`: Number of dimensions for the word embedding model\n",
    "- `window`: Number of context words to observe in each direction\n",
    "- `min_count`: Minimum frequency for words included in model\n",
    "- `sg` (Skip-Gram): '0' indicates CBOW model; '1' indicates Skip-Gram\n",
    "- `alpha`: Learning rate (initial); prevents model from over-correcting, enables finer tuning\n",
    "- `iterations`: Number of passes through dataset\n",
    "- `batch_words`: Number of words to sample from data during each pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "collecting all words and their counts\n",
      "PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "PROGRESS: at sentence #10000, processed 256693 words, keeping 9207 word types\n",
      "PROGRESS: at sentence #20000, processed 567137 words, keeping 13603 word types\n",
      "PROGRESS: at sentence #30000, processed 918759 words, keeping 17616 word types\n",
      "PROGRESS: at sentence #40000, processed 1236460 words, keeping 19579 word types\n",
      "PROGRESS: at sentence #50000, processed 1508401 words, keeping 22623 word types\n",
      "PROGRESS: at sentence #60000, processed 1715918 words, keeping 27910 word types\n",
      "PROGRESS: at sentence #70000, processed 1915119 words, keeping 30708 word types\n",
      "PROGRESS: at sentence #80000, processed 2144824 words, keeping 35822 word types\n",
      "PROGRESS: at sentence #90000, processed 2415166 words, keeping 44470 word types\n",
      "collected 51134 word types from a corpus of 2621785 raw words and 98552 sentences\n",
      "Loading a fresh vocabulary\n",
      "min_count=5 retains 17011 unique words (33% of original 51134, drops 34123)\n",
      "min_count=5 leaves 2565427 word corpus (97% of original 2621785, drops 56358)\n",
      "deleting the raw counts dictionary of 51134 items\n",
      "sample=0.001 downsamples 53 most-common words\n",
      "downsampling leaves estimated 1802467 word corpus (70.3% of prior 2565427)\n",
      "estimated required memory for 17011 words and 300 dimensions: 49331900 bytes\n",
      "resetting layer weights\n",
      "training model with 3 workers on 17011 vocabulary and 300 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
      "PROGRESS: at 1.12% examples, 194674 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 2.15% examples, 213024 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 3.36% examples, 222702 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 4.47% examples, 228070 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 5.82% examples, 223539 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 7.21% examples, 217486 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 8.43% examples, 215687 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 9.65% examples, 210600 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 10.59% examples, 207928 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 11.64% examples, 206963 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 12.67% examples, 210723 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 13.82% examples, 212895 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 15.02% examples, 214139 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 16.10% examples, 209783 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 17.07% examples, 203605 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 17.93% examples, 198907 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 18.72% examples, 195887 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 19.71% examples, 192156 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 20.47% examples, 189817 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 21.17% examples, 186500 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 21.73% examples, 182641 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 22.44% examples, 182300 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 23.50% examples, 183143 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 24.61% examples, 185696 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 25.89% examples, 185833 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 27.23% examples, 185820 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 28.38% examples, 185893 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 29.68% examples, 186204 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 30.76% examples, 186886 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 31.83% examples, 187691 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 32.84% examples, 189637 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 34.00% examples, 191259 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 35.09% examples, 191383 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 36.41% examples, 191058 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 37.49% examples, 189599 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 38.45% examples, 189001 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 39.65% examples, 188520 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 40.56% examples, 188195 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 41.46% examples, 187546 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 42.35% examples, 188374 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 43.38% examples, 188618 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 44.39% examples, 189420 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 45.45% examples, 188932 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 46.11% examples, 186869 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 47.23% examples, 186033 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 48.38% examples, 186113 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 49.60% examples, 185852 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 50.56% examples, 185847 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 51.26% examples, 184525 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 52.00% examples, 184074 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 52.84% examples, 184396 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 53.83% examples, 184642 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 54.88% examples, 185039 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 56.03% examples, 184491 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 57.23% examples, 184132 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 58.24% examples, 183656 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 58.98% examples, 182995 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 59.90% examples, 181814 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 60.85% examples, 181748 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 61.88% examples, 182035 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 62.63% examples, 181982 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 63.44% examples, 181482 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 64.15% examples, 181199 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 65.24% examples, 181383 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 66.58% examples, 181488 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 67.90% examples, 181628 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 69.08% examples, 182047 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 70.19% examples, 181830 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 71.30% examples, 182072 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 72.31% examples, 182869 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 73.47% examples, 183439 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 74.58% examples, 184332 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 75.89% examples, 184431 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 77.08% examples, 184215 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 78.24% examples, 184162 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 79.08% examples, 183734 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 80.11% examples, 183378 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 80.96% examples, 183023 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 81.74% examples, 182573 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 82.59% examples, 182912 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 83.63% examples, 183130 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 84.58% examples, 183446 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 85.94% examples, 183593 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 87.23% examples, 183549 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 88.34% examples, 183548 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 89.25% examples, 183138 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 90.22% examples, 182933 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 91.18% examples, 182727 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 92.00% examples, 182662 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 92.89% examples, 182983 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 93.91% examples, 183278 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 95.09% examples, 183597 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 96.36% examples, 183570 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 97.67% examples, 183501 words/s, in_qsize 0, out_qsize 0\n",
      "PROGRESS: at 98.78% examples, 183801 words/s, in_qsize 0, out_qsize 0\n",
      "worker thread finished; awaiting finish of 2 more threads\n",
      "PROGRESS: at 99.98% examples, 183696 words/s, in_qsize 1, out_qsize 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "worker thread finished; awaiting finish of 1 more threads\n",
      "worker thread finished; awaiting finish of 0 more threads\n",
      "training on 26217850 raw words (18025186 effective words) took 98.1s, 183712 effective words/s\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import gutenberg\n",
    "from gensim import models\n",
    "\n",
    "# Training word2vec model on Gutenberg corpus. This may take a few minutes.\n",
    "model = models.Word2Vec(gutenberg.sents(), size = 300, window = 5, min_count =5, sg = 0, alpha = 0.025, iter=10, batch_words = 10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](http://multithreaded.stitchfix.com/assets/images/blog/vectors.gif)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The word vectors are directions in space and can encode relationships between words.  \n",
    "\n",
    "The proximity of words to each other can be calculated through their cosine similarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('girl', 0.7028565406799316),\n",
       " ('child', 0.7010887861251831),\n",
       " ('lad', 0.6638487577438354),\n",
       " ('Farmer', 0.6618794798851013),\n",
       " ('maid', 0.6375283002853394),\n",
       " ('gentleman', 0.6298720836639404),\n",
       " ('dog', 0.6218686103820801),\n",
       " ('fellow', 0.6167354583740234),\n",
       " ('carpenter', 0.6009632349014282),\n",
       " ('lover', 0.6000135540962219)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['boy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('cattle', 0.663178026676178),\n",
       " ('bread', 0.6592459678649902),\n",
       " ('fruits', 0.6554306745529175),\n",
       " ('flocks', 0.6438690423965454),\n",
       " ('grapes', 0.6433919668197632),\n",
       " ('fruit', 0.6313921809196472),\n",
       " ('money', 0.6305223107337952),\n",
       " ('wine', 0.6153505444526672),\n",
       " ('meat', 0.6142132878303528),\n",
       " ('sheep', 0.6097452044487)]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['food'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Cecilia', 0.4503798186779022),\n",
       " ('treacherous', 0.4389117360115051),\n",
       " ('Mary', 0.4288462996482849),\n",
       " ('Anne', 0.42407548427581787),\n",
       " ('particularly', 0.41419678926467896),\n",
       " ('suited', 0.3981710970401764),\n",
       " ('Emma', 0.39410048723220825),\n",
       " ('diplomacy', 0.3867933750152588),\n",
       " ('Fairfax', 0.3854631185531616),\n",
       " ('grateful', 0.3798571228981018)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['she','her','hers','herself'], negative=['he','him','his','himself'])\n",
    "# she + her + hers + herself - he - him - his - himself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's limit ourselves to top 50 words that related to food to visualize how they relate in vector space\n",
    "f_tokens = [token for token,weight in model.wv.most_similar(positive=['food'], topn=50)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import pairwise\n",
    "vectors = [model.wv[word] for word in f_tokens]\n",
    "dist_matrix = pairwise.pairwise_distances(vectors, metric='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import MDS\n",
    "mds = MDS(n_components = 2, dissimilarity='precomputed')\n",
    "embeddings = mds.fit_transform(dist_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz4AAAJCCAYAAAABXBLWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4VtXdt/1zC8gMWqFWUBvsLWMmJAlgUBBkUBAUg6AB\nRQRFHtDqLYXeWkVKW6u0UBBNncABLIJFkVK1yCDjDQmGGUR4U1B5lLlEQEjY7x+U64ECMoUErpyf\n4+jhde299lq/lVbKN3vvtYIwDJEkSZKkaHZBURcgSZIkSWebwUeSJElS1DP4SJIkSYp6Bh9JkiRJ\nUc/gI0mSJCnqGXwkSZIkRT2DjyRJkqSoZ/CRJEmSFPUMPpIkSZKiXsmiLuB4qlSpEsbExBR1GZIk\nSZLOYVlZWVvCMKx6onbnbPCJiYkhMzOzqMuQJEmSdA4LguCfJ9POR90kSZIkRT2DjyRJkqSoZ/CR\nJEmSFPUMPpIkSZKinsFHkiRJUtQz+EiSJEmKegYfSZIkSVHP4CNJkiQp6hl8JEmSJEU9g48kSZKk\nqGfwkSRJkhT1DD6SJEmSop7BR5IkSVLUM/hIkiRJinoGH0mSJElRz+AjSZIkKeoZfCRJkiRFPYOP\nJEmSpKhn8JEk6RQ8+eSTTJs2rajLkCSdopJFXYAkSeeTwYMHF3UJkqTT4B0fSVKxlpOTQ+3atUlP\nT6dOnTqkpaWxe/duBg8eTHJyMrGxsdx///2EYQhA9+7dmThxIgAxMTE89dRTXHPNNcTFxbF69WoA\nZs2aRWJiIomJidSvX59du3YV2fwkSQcZfCRJxd6aNWvo06cPq1atolKlSrzwwgv07duXRYsWsXz5\ncvbs2cOUKVOOeW2VKlVYvHgxDz74IEOHDgVg6NChjBo1iuzsbGbPnk3ZsmULczqSpGMw+EiSir0r\nrriC1NRUALp27cqcOXOYMWMGDRs2JC4ujunTp7NixYpjXtuxY0cAGjRoQE5ODgCpqak8+uijjBgx\ngh07dlCypE+WS1JRM/hIkoq9IAiO+t6nTx8mTpzIsmXL6NWrF3v37j3mtaVLlwagRIkS5OXlATBw\n4EBeeeUV9uzZQ2pqauQROElS0TH4SJKKvQ0bNjB//nwAxo0bR5MmTYCDj7Hl5uZG3uk5WevWrSMu\nLo4BAwaQnJxs8JGkc4D33iVJxV6tWrUYNWoUPXr0oG7dujz44INs376d2NhYfvKTn5CcnHxK/Q0f\nPpwZM2ZwwQUXUK9ePW666aazVLkk6WQFh1apOdckJSWFmZmZRV2GJCnK5eTk0K5dO5YvX17UpUiS\nTkMQBFlhGCadqJ2PukmSJEmKegUSfIIgaBMEwZogCL4IgmDgcdrcEQTByiAIVgRBMK4gxpUk6UzF\nxMR4t0eSioEzfscnCIISwCigJfAlsCgIgslhGK48rM3VwC+B1DAMtwdB8OMzHVeSJEmSTlZB3PFJ\nAb4Iw3B9GIb7gL8AHf6jTS9gVBiG2wHCMPy2AMaVJEmSpJNSEMGnOrDxsO9f/vvY4WoCNYMgmBsE\nwYIgCNoUwLiSJEmSdFIKaznrksDVQDPgcuDTIAjiwjDccXijIAjuB+4HuPLKKwupNEmSJEnRriDu\n+HwFXHHY98v/fexwXwKTwzDcH4bh/wd8zsEgdIQwDF8KwzApDMOkqlWrFkBpkiRJklQwwWcRcHUQ\nBDWCILgQ6AJM/o8273Hwbg9BEFTh4KNv6wtgbEmSJEk6oTMOPmEY5gF9gY+AVcA7YRiuCIJgcBAE\n7f/d7CNgaxAEK4EZQP8wDLee6diSJEmSdDKCMAyLuoZjSkpKCjMzM4u6DEmSJEnnsCAIssIwTDpR\nuwLZwFSSJEmSzmUGH0mSJElRz+AjSZIkKeoZfCRJkiRFPYOPJEmSpKhn8JEkSZIU9Qw+kiRJkqKe\nwUeSJElS1DP4SJIkSYp6Bh9JkiRJUc/gI0mSJCnqGXwkSZIkRT2DjyRJkqSoZ/CRJEmSFPUMPpIk\nSZKinsFHkiRJUtQz+EiSJEmKegYfSZIkSVHP4CNJkiQp6hl8JEmSJEU9g48kSZKkqGfwkSRJkhT1\nDD6SJEmSop7BR5IkSVLUM/hIkiRJinoGH0mSJElRz+AjSZIkKeoZfCRJkiRFPYOPJEmSpKhn8JEk\nSZIU9Qw+kiRJkqKewUeSJElS1DP4SJIkSYp6Bh9JkiRJUc/gI0mSJCnqGXwkSZIkRT2DjyRJkqSo\nZ/CRJEnFxs0338yOHTuOe7579+5MnDixECuSVFhKFnUBkiRJhSEMQ6ZMmcIFF/h7X6k48t98SZIU\ntXJycqhVqxZ33303sbGxlChRgi1btgDwxhtvEB8fT0JCAt26dYtc8+mnn3Lttddy1VVXHXH357nn\nniM5OZn4+HieeuopAL777jvatm1LQkICsbGxjB8/vnAnKOmkecdHkiRFtbVr1/L666/TqFEjYmJi\nAFixYgVDhgxh3rx5VKlShW3btkXab9q0iTlz5rB69Wrat29PWloaH3/8MWvXrmXhwoWEYUj79u35\n9NNP2bx5M9WqVeNvf/sbADt37iyKKUo6Cd7xkSRJUe2nP/0pjRo1OuLY9OnT6dSpE1WqVAHgRz/6\nUeTcrbfeygUXXEDdunX55ptvAPj444/5+OOPqV+/Ptdccw2rV69m7dq1xMXF8Y9//IMBAwYwe/Zs\nKleuXHgTk3RKvOMjSZKiWvny5U+pfenSpSOfwzCM/POXv/wlDzzwwFHtFy9ezNSpU3niiSdo0aIF\nTz755JkVLOms8I6PJEkqdpo3b86ECRPYunUrwBGPuh1L69atee2118jNzQXgq6++4ttvv+Xrr7+m\nXLlydO3alf79+7N48eKzXruk0+MdH0mSVOzUq1ePxx9/nKZNm1KiRAnq16/PmDFjjtu+VatWrFq1\nisaNGwNQoUIF3nrrLb744gv69+/PBRdcQKlSpXjxxRcLaQaSTlVw6BbuuSYpKSnMzMws6jIkFbIx\nY8aQmZnJ888/X9SlSJKk80AQBFlhGCadqJ2PukmSJEmKegYfSQXmWPtZfPLJJ9SvX5+4uDh69OjB\n999/D0BMTExkL43MzEyaNWt2VH8TJkwgNjaWhIQErr/++sKciiRJijK+4yOpwHz44YdH7WcRGxvL\nJ598Qs2aNbn77rt58cUX+fnPf35S/Q0ePJiPPvqI6tWrs2PHjrNZuiRJinLe8ZFUYP5zP4ucnBxq\n1KhBzZo1Abjnnnv49NNPT7q/1NRUunfvzssvv0x+fv7ZKluSJBUDBh9JBaZmzZosXryYuLg4nnji\nCd57773jti1ZsiQHDhwAYO/evcdsk5GRwZAhQ9i4cSMNGjSILDsrSZJ0qgw+kgrMf+5nMX/+fHJy\ncvjiiy8AePPNN2natClw8B2frKwsAN59991j9rdu3ToaNmzI4MGDqVq1Khs3biyciUiSpKjjOz6S\nCsyyZcuO2s9i586ddOrUiby8PJKTk+nduzcATz31FPfddx+/+tWvjrmwAUD//v1Zu3YtYRjSokUL\nEhISCnE2kiQpmriPjyRJkqTzlvv4SJIkSdK/GXwkSZIkRT2DjyRJkqSoZ/CRJEmSFPUMPpIkSZKi\nnsFHkiRJUtQz+EiSJEmKegYfSZIkSVHP4CNJkiQp6hl8JKkIZWRk8MYbbwDQvXt3Jk6cWMQVSZIU\nnUoWdQGSVJz17t27qEuQJKlY8I6PJJ2m7777jrZt25KQkEBsbCzjx48nJiaGX/ziF8TFxZGSksIX\nX3wBQE5ODs2bNyc+Pp4WLVqwYcMGAAYNGsTQoUOLchqSJBULBh9JOk0ffvgh1apVY8mSJSxfvpw2\nbdoAULlyZZYtW0bfvn35+c9/DkC/fv245557WLp0Kenp6Tz00ENFWbokScWOwUeSTlNcXBz/+Mc/\nGDBgALNnz6Zy5coA3HnnnZF/zp8/H4D58+dz1113AdCtWzfmzJlTNEVLklRM+Y6PJJ2mmjVrsnjx\nYqZOncoTTzxBixYtAAiCINLm8M+SJKnoeMdHkk7T119/Tbly5ejatSv9+/dn8eLFAIwfPz7yz8aN\nGwNw7bXX8pe//AWAsWPHct111xVN0ZIkFVPe8ZGk07Rs2TL69+/PBRdcQKlSpXjxxRdJS0tj+/bt\nxMfHU7p0ad5++20ARo4cyb333stzzz1H1apVGT16dBFXL0lS8RKEYVjUNRxTUlJSmJmZWdRlSNIp\niYmJITMzkypVqhR1KZIkFQtBEGSFYZh0onY+6iZJkiQp6vmomyQVoJycnKIuQZIkHYN3fCRJkiRF\nPYOPJEmSpKhn8JEkSZIU9Qw+kiRJkqKewUeSJElS1DP4SJIkSYp6Bh9JkiRJUc/gI0mSJCnqGXwk\nSZIkRT2DjyRJkqSoVyDBJwiCNkEQrAmC4IsgCAb+QLvbgyAIgyBIKohxJUmSJOlknHHwCYKgBDAK\nuAmoC9wZBEHdY7SrCDwM/O+ZjilJkiRJp6Ig7vikAF+EYbg+DMN9wF+ADsdo92vg98DeAhhTkiRJ\nkk5aQQSf6sDGw75/+e9jEUEQXANcEYbh336ooyAI7g+CIDMIgszNmzcXQGmSJEmSVAiLGwRBcAHw\nR+C/T9Q2DMOXwjBMCsMwqWrVqme7NEmSJEnFREEEn6+AKw77fvm/jx1SEYgFZgZBkAM0Aia7wIEk\nSSqOmjVrRmZmZlGXIRU7BRF8FgFXB0FQIwiCC4EuwORDJ8Mw3BmGYZUwDGPCMIwBFgDtwzD033hJ\nkiRJheKMg08YhnlAX+AjYBXwThiGK4IgGBwEQfsz7V+SJOlckJOTQ2xsbOT70KFDGTRoEM2aNWPA\ngAGkpKRQs2ZNZs+eDUB+fj6PPfYYsbGxxMfHM3LkyKP6/Pjjj2ncuDHXXHMNnTp1Ijc3F4DBgweT\nnJxMbGws999/P2EYAhx3rDFjxtCxY0fatGnD1VdfzS9+8YsTjjFw4EDq1q1LfHw8jz32GAATJkwg\nNjaWhIQErr/++rPwU5SKToG84xOG4dQwDGuGYfizMAx/8+9jT4ZhOPkYbZt5t0eSJEWTvLw8Fi5c\nyPDhw3n66acBeOmll8jJySE7O5ulS5eSnp5+xDVbtmxhyJAhTJs2jcWLF5OUlMQf//hHAPr27cui\nRYtYvnw5e/bsYcqUKT84FkB2djbjx49n2bJljB8/no0bNx53jK1btzJp0iRWrFjB0qVLeeKJJ4CD\ngeujjz5iyZIlTJ581F/jpPNayaIuQJIknTtycnJo164dy5cvL+pSzisdO3YEoEGDBuTk5AAwbdo0\nevfuTcmSB/+69aMf/eiIaxYsWMDKlStJTU0FYN++fTRu3BiAGTNm8Oyzz7J79262bdtGvXr1uOWW\nW447FkCLFi2oXLkyAHXr1uWf//wnO3bsOOYYlStXpkyZMtx33320a9eOdu3aAZCamkr37t254447\nIuNI0cLgI0mSzjl5eXmRwHCuKFmyJAcOHIh837v3/21NWLp0aQBKlChBXl7eSfUXhiEtW7bk7bff\nPuL43r176dOnD5mZmVxxxRUMGjTopMY6dPzwc8cbA2DhwoV88sknTJw4keeff57p06eTkZHB//7v\n//K3v/2NBg0akJWVxSWXXHJS85HOdWd9OWtJknR+yc/Pp1evXtSrV49WrVqxZ8+eI1Yi27JlCzEx\nMcAPv1vy6quvUrNmTVJSUujVqxd9+/YFYPPmzdx+++0kJyeTnJzM3LlzARg0aBDdunUjNTWVbt26\nFe6kT8Kll17Kt99+y9atW/n++++PePzsWFq2bMmf//znSDjZtm3bEecbNWrE3Llz+eKLLwD47rvv\n+PzzzyMhp0qVKuTm5jJx4sTTrvl4Y+Tm5rJz505uvvlmhg0bxpIlSwBYt24dDRs2ZPDgwVStWpWN\nGzf+UPfSeeXc+lWKJEkqcmvXruXtt9/m5Zdf5o477uDdd9/9wfbZ2dl89tlnlC5dmlq1atGvXz9K\nlCjBr3/9axYvXkzFihVp3rw5CQkJADz88MM88sgjNGnShA0bNtC6dWtWrVoFwMqVK5kzZw5ly5Y9\n6/M8VaVKleLJJ58kJSWF6tWrU7t27R9s37NnTz7//HPi4+MpVarUEeEPoGrVqowZM4Y777yT77//\nHoAhQ4ZQs2ZNevXqRWxsLD/5yU9ITk4+7ZqPN0bFihXp0KEDe/fuJQzDyLtF/fv3Z+3atYRhSIsW\nLSL/nUnRIDi0Ssi5JikpKXSNe0mSCldOTg4tW7Zk7dq1APz+979n//79TJs2jaFDh5KUlMSWLVtI\nSkoiJyeHMWPGMHfuXF5++WUAbrrpJh5//HG2bNnCpEmTeP311wEYMWIEn3/+Oc8//zw//vGPqVat\nWmTMzZs3s2bNGoYOHUoQBDz11FOFP3FJ560gCLLCMDzhHqHe8ZEkSUf4z3dF9uzZc8T7LYe/b3Ks\n9id6x+XAgQMsWLCAMmXKHHWufPnyZ1K6JB2X7/hIkqQTiomJISsrC+Ck3jlJTk5m1qxZbN++nby8\nvCMel2vVqtURe9pkZ2cXfMGS9B8MPpIk6YQee+wxXnzxRerXr8+WLVtO2L569er8z//8DykpKaSm\nphITExNZannEiBFkZmYSHx9P3bp1ycjIONvlS5Lv+EiSpLMjNzeXChUqkJeXx2233UaPHj247bbb\nirosSVHmZN/x8Y6PJEk6KwYNGkRiYiKxsbHUqFGDW2+9tahLklSMubiBJEk6K4YOHVrUJUhShHd8\nJEmSJEU9g48knYScnBxiY2OLugxJknSaDD6SJEmSop7BR1JU+vWvf02tWrVo0qQJd955J0OHDiU7\nO5tGjRoRHx/Pbbfdxvbt2wGOezwrK4uEhAQSEhIYNWpUpO8VK1aQkpJCYmIi8fHxkR3uJUnSucvg\nIynqLFq0iHfffZclS5bw97//nUNL49999938/ve/Z+nSpcTFxfH000//4PF7772XkSNHsmTJkiP6\nz8jI4OGHHyY7O5vMzEwuv/zywp2gJEk6ZQYfSVFn7ty5dOjQgTJlylCxYkVuueUWvvvuO3bs2EHT\npk0BuOeee/j000/ZuXPnMY/v2LGDHTt2cP311wPQrVu3SP+NGzfmt7/9Lb///e/55z//SdmyZQt/\nkpIk6ZQYfCTpFN11111MnjyZsmXLcvPNNzN9+vSiLkmSJJ2AwUdS1ElNTeWDDz5g79695ObmMmXK\nFMqXL8/FF1/M7NmzAXjzzTdp2rQplStXPubxiy66iIsuuog5c+YAMHbs2Ej/69ev56qrruKhhx6i\nQ4cOLF26tPAnKUmSTokbmEqKOsnJybRv3574+HguvfRS4uLiqFy5Mq+//jq9e/dm9+7dXHXVVYwe\nPRrguMdHjx5Njx49CIKAVq1aRfp/5513ePPNNylVqhQ/+clP+J//+Z8imackSTp5QRiGRV3DMSUl\nJYWHXkiWpFOVm5tLhQoV2L17N9dffz0vvfQS11xzTVGXJUmSClgQBFlhGCadqJ13fCRFpfvvv5+V\nK1eyd+9e7rnnHkOPJEnFnMFHUlQaN25cUZcgSZLOIS5uIEmSJCnqGXwkSZIkRT2Dj1QM5OfnF3UJ\nkiRJRcrgI53ncnJyqF27Nunp6dSpU4e0tDR2795NTEwMAwYM4JprrmHChAmsW7eONm3a0KBBA667\n7jpWr17Nrl27qFGjBvv37wfgX//61xHfJUmSooXBR4oCa9asoU+fPqxatYpKlSrxwgsvAHDJJZew\nePFiunTpwv3338/IkSPJyspi6NCh9OnTh4oVK9KsWTP+9re/AfCXv/yFjh07UqpUqaKcjiRJUoFz\nVTcpClxxxRWkpqYC0LVrV0aMGAFA586dgYN72sybN49OnTpFrvn+++8B6NmzJ88++yy33noro0eP\n5uWXXy7k6iVJks4+g48UBYIgOOb38uXLA3DgwAEuuugisrOzj7o2NTWVnJwcZs6cSX5+PrGxsWe/\nYEmSpELmo25SFNiwYQPz588HDu5f06RJkyPOV6pUiRo1ajBhwgQAwjBkyZIlkfN33303d911F/fe\ne2/hFS1JklSIDD5SFKhVqxajRo2iTp06bN++nQcffPCoNmPHjuXVV18lISGBevXq8f7770fOpaen\ns337du68887CLFuSJKnQ+KibFAVKlizJW2+9dcSxnJycI77XqFGDDz/88JjXz5kzh7S0NC666KKz\nVaIkSVKRMvhIxVy/fv34+9//ztSpU4u6FEmSpLMmCMOwqGs4pqSkpDAzM7Ooy5AkSZJ0DguCICsM\nw6QTtfMdH0mSJElRz+AjSZIkKeoZfCRJkiRFPYOPJEmSpKhn8JEkSZIU9Qw+kiRJkqKewUeSJElS\n1DP4SJIkSYp6Bh9JkiRJUc/gI0mSJCnqGXwkSZIkRT2DjyRJkqSoZ/AphkaMGEGdOnVIT08/5vkx\nY8bQt29fALp3787EiRMLszxJkiSpwJUs6gJU+F544QWmTZvG5ZdfXtSlSJIkSYXCOz7FTO/evVm/\nfj033XQTf/jDH7j11luJj4+nUaNGLF269Aev/dWvfkX37t3Jz89n4MCB1K1bl/j4eB577LFCql6S\nJEk6PQafYiYjI4Nq1aoxY8YMcnJyqF+/PkuXLuW3v/0td99993Gv69+/P5s3b2b06NHs2LGDSZMm\nsWLFCpYuXcoTTzxRiDOQJEmSTp3BpxibM2cO3bp1A6B58+Zs3bqVf/3rX0e1+/Wvf83OnTvJyMgg\nCAIqV65MmTJluO+++/jrX/9KuXLlCrt0SZIk6ZQYfHRCycnJZGVlsW3bNgBKlizJwoULSUtLY8qU\nKbRp06aIK5QkSZJ+mMGnGLvuuusYO3YsADNnzqRKlSpUqlTpqHZt2rRh4MCBtG3bll27dpGbm8vO\nnTu5+eabGTZsGEuWLCns0iVJkqRT4qpuxdigQYPo0aMH8fHxlCtXjtdff/24bTt16sSuXbto3749\n48aNo0OHDuzdu5cwDPnjH/9YiFVLkiRJpy4Iw7CoazimpKSkMDMzs6jLkCRJknQOC4IgKwzDpBO1\n81E3SZKkKHNos/Lq1atHNiU/FTNnzqRdu3ZnoTKp6PiomyRJUpQ5tFn5tGnT8Aka6SDv+EiSJEWR\nwzcr3759e+T45s2buf3220lOTiY5OZm5c+cCMGvWLBITE0lMTKR+/frs2rULgNzcXNLS0qhduzbp\n6emcq69HSCfL4CNJkhRFDt+s/OKLL44cf/jhh3nkkUdYtGgR7777Lj179gRg6NChjBo1iuzsbGbP\nnk3ZsmUB+Oyzzxg+fDgrV65k/fr1kaAkna981E2SJKkYmDZtGitXrox8/9e//kVubi6pqak8+uij\npKen07FjRy6//HIAUlJSIp8TExPJycmhSZMmRVK7VBAMPpIkScXAgQMHWLBgAWXKlDni+KG9+qZO\nnUpqaiofffQRAKVLl460KVGiBHl5eYVar1TQfNRNkiSpGGjVqhUjR46MfM/OzgZg3bp1xMXFMWDA\nAJKTk1m9enVRlSidVQYfSZKkYmDEiBFkZmYSHx9P3bp1ycjIAGD48OHExsYSHx9PqVKluOmmm4q4\nUunscANTSZIkSectNzCVipGZM2cyb968yPf33nvviBdYu3fvzsSJE4uiNEmSpHOCwUeKAicKPpIk\nScWdwUc6h73xxhvEx8eTkJBAt27d+OCDD2jYsCH169fnxhtv5JtvviEnJ4eMjAyGDRtGYmIis2bN\nYvLkyfTv35/ExETWrVt3RJ9ZWVk0bdqUBg0a0Lp1azZt2lREs5MkSSo8LmctnaNWrFjBkCFDmDdv\nHlWqVGHbtm0EQcCCBQsIgoBXXnmFZ599lj/84Q/07t2bChUq8NhjjwHQvn172rVrR1pa2hF97t+/\nn379+vH+++9TtWpVxo8fz+OPP85rr71WFFOUJEkqNAYf6Rw1ffp0OnXqRJUqVQD40Y9+xLJly+jc\nuTObNm1i37591KhR45T6XLNmDcuXL6dly5YA5Ofnc9lllxV47ZIkSecag490HunXrx+PPvoo7du3\nZ+bMmQwaNOiUrg/DkHr16jF//vyzU6AkSdI5ynd8pHNU8+bNmTBhAlu3bgVg27Zt7Ny5k+rVqwPw\n+uuvR9pWrFiRXbt2Hff7IbVq1WLz5s2R4LN//35WrFhxNqchSZJ0TjD4SOeoevXq8fjjj9O0aVMS\nEhJ49NFHGTRoEJ06daJBgwaRR+AAbrnlFiZNmkRiYiKzZ8+mS5cuPPfcc9SvX/+IxQ0uvPBCJk6c\nyIABA0hISCAxMfGI1eAkSZKilRuYSpIkSTpvuYGpJEmSJP2bwUeSJElS1DP4SJIkSYp6Bh9JkiRJ\nUc/gI0mSJCnqGXwkSSqmKlSocFb6HTNmDH379j0rfUvS6TL4SJIkSYp6Bh9JkoqBW2+9lQYNGlCv\nXj1eeumlyPFHHnmEevXq0aJFCzZv3gxAs2bNOLSX3pYtW4iJiQEO3snp2LEjbdq04eqrr+YXv/hF\npJ/Ro0dTs2ZNUlJSmDt3buT4Bx98QMOGDalfvz433ngj33zzDQCDBg2iR48eNGvWjKuuuooRI0ZE\nrnnjjTeIj48nISGBbt26AbB582Zuv/12kpOTSU5OPmIMSToZBh9JkoqB1157jaysLDIzMxkxYgRb\nt27lu+++IykpiRUrVtC0aVOefvrpE/aTnZ3N+PHjWbZsGePHj2fjxo1s2rSJp556irlz5zJnzhxW\nrlwZad+kSRMWLFjAZ599RpcuXXj22Wcj51avXs1HH33EwoULefrpp9m/fz8rVqxgyJAhTJ8+nSVL\nlvCnP/0JgIcffphHHnmERYsW8e6779KzZ8+C/yFJimoli7oASZJ09o0YMYJJkyYBsHHjRtauXcsF\nF1xA586dAejatSsdO3Y8YT8tWrSgcuXKANStW5d//vOfbNmyhWbNmlG1alUAOnfuzOeffw7Al19+\nSefOndm0aRP79u2jRo0akb7atm1L6dKlKV26ND/+8Y/55ptvmD59Op06daJKlSoA/OhHPwJg2rRp\nRwSqf/3rX+Tm5p6195QkRZ8CueMTBEGbIAjWBEHwRRAEA49x/tEgCFYGQbA0CIJPgiD4aUGMK0mS\nTmzmzJlZQVDsAAAgAElEQVRMmzaN+fPns2TJEurXr8/evXuPahcEAQAlS5bkwIEDAEe1K126dORz\niRIlyMvL+8Gx+/XrR9++fVm2bBl//vOfj+jvVPo6cOAACxYsIDs7m+zsbL766itDTzFz7bXXFnUJ\nOs+dcfAJgqAEMAq4CagL3BkEQd3/aPYZkBSGYTwwEXgWSZJUKHbu3MnFF19MuXLlWL16NQsWLAAO\nhomJEycCMG7cOJo0aQJATEwMWVlZAJHzP6Rhw4bMmjWLrVu3sn//fiZMmHDE2NWrVwfg9ddfP2Ff\nzZs3Z8KECWzduhWAbdu2AdCqVStGjhwZaZednX3CvhRd5s2bd8Z9nCioK7oVxB2fFOCLMAzXh2G4\nD/gL0OHwBmEYzgjDcPe/vy4ALi+AcSVJ0klo06YNeXl51KlTh4EDB9KoUSMAypcvz8KFC4mNjWX6\n9Ok8+eSTADz22GO8+OKL1K9fny1btpyw/8suu4xBgwbRuHFjUlNTqVOnTuTcoEGD6NSpEw0aNIg8\nvvZD6tWrx+OPP07Tpk1JSEjg0UcfBQ4+qpeZmUl8fDx169YlIyPjdH4UOo8dusM3c+ZMmjVrRlpa\nGrVr1yY9PZ0wDAFYtGgR1157LQkJCaSkpLBr1y7GjBlD+/btad68OS1atADgueeeIzk5mfj4eJ56\n6qnIGMdaBCQ/P5/u3bsTGxtLXFwcw4YNA2DdunW0adOGBg0acN1117F69erC/HHodIRheEb/AdKA\nVw773g14/gfaPw88cZxz9wOZQOaVV14ZSpJUFIYNGxZ+9913RV2GpMOUL18+DMMwnDFjRlipUqVw\n48aNYX5+ftioUaNw9uzZ4ffffx/WqFEjXLhwYRiGYbhz585w//794ejRo8Pq1auHW7duDcMwDD/6\n6KOwV69e4YEDB8L8/Pywbdu24axZs8IwDCNtdu/eHdarVy/csmVLmJmZGd54442ROrZv3x6GYRg2\nb948/Pzzz8MwDMMFCxaEN9xwQ+H8IHQUIDM8idxSqIsbBEHQFUgCmh7rfBiGLwEvASQlJYWFWJok\nSRHDhw+na9eulCtX7oz7ysvLo2RJ1xKSClJKSgqXX37wAaLExERycnKoXLkyl112GcnJyQBUqlQp\n0r5ly5aRhTI+/vhjPv74Y+rXrw9Abm4ua9eu5frrrz/mIiC1atVi/fr19OvXj7Zt29KqVStyc3OZ\nN28enTp1iozx/fffF8rcdfoK4k/ir4ArDvt++b+PHSEIghuBx4GmYRj6vwxJ0hn57rvvuOOOO/jy\nyy/Jz8/nV7/6FWvWrOGDDz5gz549XHvttfz5z38mCAKaNWtGw4YNmTFjBjt27ODVV1/luuuuIz8/\nnwEDBvDhhx9ywQUX0KtXL8Iw5Ouvv+aGG26gSpUqzJgxgwoVKpCbmwscfOdlypQpjBkzhg8++IAh\nQ4awb98+LrnkEsaOHcull17KoEGDWLduHevXr+fKK6/kq6++YsSIESQmJgIHl3geNWoUCQkJRfkj\nlM5bp7rIRvny5SOfwzDkl7/8JQ888MARbQ5fBKRcuXI0a9aMvXv3cvHFF7NkyRI++ugjMjIyeOed\ndxg+fDgXXXSR75qdZwriHZ9FwNVBENQIguBCoAsw+fAGQRDUB/4MtA/D8NsCGFOSVMx9+OGHVKtW\njSVLlrB8+XLatGlD3759WbRoEcuXL2fPnj1MmTIl0j4vL4+FCxcyfPjwyH41L730Ejk5OWRnZ7N0\n6VLS09N56KGHqFatGjNmzGDGjBk/WMMP7VGzcuVKpk2bxttvv819993HmDFjAPj888/Zu3evoUcq\nYLVq1WLTpk0sWrQIgF27dh0zELVu3ZrXXnst8suMr776im+//fa4i4Bs2bKFAwcOcPvttzNkyBAW\nL15MpUqVqFGjRmQhjzAMWbJkSSHNVKfrjINPGIZ5QF/gI2AV8E4YhiuCIBgcBEH7fzd7DqgATAiC\nIDsIgsnH6S6q5OTkEBsbe1b6zs7OZurUqWelb0k6H8TFxfGPf/yDAQMGMHv2bCpXrsyMGTNo2LAh\ncXFxTJ8+nRUrVkTaH9qjpkGDBuTk5AAH94Z54IEHIo+iHXoU5mR9+eWXtG7dmri4OJ577rkjxmvf\nvj1ly5YFoFOnTkyZMoX9+/fz2muv0b179zOYuaRjufDCCxk/fjz9+vUjISGBli1bHnPZ9latWnHX\nXXfRuHFj4uLiSEtLY9euXcddBOSrr76iWbNmJCYm0rVrV373u98BMHbsWF599VUSEhKoV68e77//\nfqHOV6euQB46DsNwKjD1P449edjnGwtiHP0/2dnZZGZmcvPNNxd1KZJUJGrWrMnixYuZOnUqTzzx\nBC1atGDUqFFkZmZyxRVXMGjQoGPuGXMyj8X8p0P728CR+9r069ePRx99lPbt2zNz5kwGDRoUOXf4\nozXlypWjZcuWvP/++7zzzjuRpaIlnbxDd2iaNWtGs2bNIseff/75yOfk5OTInZpDunfvftQvGx5+\n+GEefvjho8b4+9//fsyxFy9efNSxGjVq8OGHH55s+ToHFMgGpjq+vLw80tPTqVOnDmlpaezevZus\nrCyaNm1KgwYNaN26NZs2bQLg5ZdfJjk5mYSEBG6//XZ27z64AviECROIjY0lISGB66+/nn379vHk\nk08yfvx4EhMTGT9+PLNmzSIxMZHExETq16/Prl27inLaknTWff3115QrV46uXbvSv3//yF9MqlSp\nQm5u7kntP9OyZUv+/Oc/R4LQoT1jKlaseMSfo5deeimrVq3iwIEDkRef4dT2qOnZsycPPfQQycnJ\nXHzxxac2WUnSGTP4nGVr1qyhT58+rFq1ikqVKjFq1Cj69evHxIkTycrKokePHjz++OPAwccwFi1a\nxJIlS6hTpw6vvvoqAIMHD+ajjz5iyZIlTJ48mQsvvJDBgwfTuXNnsrOz6dy5M0OHDmXUqFFkZ2cz\ne/bsyOMVkhStli1bRkpKComJiTz99NM88cQT9OrVi9jYWFq3bh1Z2emH9OzZkyuvvJL4+HgSEhIY\nN24cAPfffz9t2rThhhtuAOCZZ56hXbt2XHvttVx22WWR609lj5oGDRpQqVIl7r333jOYtSTpdAVh\neG6uGp2UlBRmZmYWdRlnJCcnh+uvv54NGzYAMH36dH7729+ycOFCrrrqKuDgpliXXXYZH3/8MbNm\nzeKJJ55gx44d5Obm0rp1azIyMujduzfr1q3jjjvuoGPHjlxyySWMGTOGzMzMyO3dZ555hkmTJpGe\nnk7Hjh0jSzxKks4NX3/9Nc2aNWP16tVccIG/d5SkghIEQVYYhkknauefvGfZ4c+Fw8HHJ+rVq0d2\ndjbZ2dksW7aMjz/+GDj4DOrzzz/PsmXLeOqppyLPkWdkZDBkyBA2btxIgwYN2Lp161HjDBw4kFde\neYU9e/aQmprq7sGSdA554403aNiwIb/5zW8MPZJURPzT9yzbsGED8+fPB2DcuHE0atSIzZs3R47t\n378/sgrQrl27uOyyy9i/fz9jx46N9LFu3ToaNmzI4MGDqVq1Khs3bjzq+fN169YRFxfHgAEDSE5O\nNvhI0jnk7rvvZuPGjUdsdihJKlwGn7OsVq1ajBo1ijp16rB9+/bI+z0DBgwgISGBxMRE5s2bB8Cv\nf/1rGjZsSGpqKrVr14700b9/f+Li4oiNjeXaa68lISGBG264gZUrV0YWNxg+fDixsbHEx8dTqlQp\nbrrppqKasiRJknTO8R0fSZIkSect3/GRJEmSpH8z+EiSJEmKegYfSYpyI0aMoE6dOlx88cU888wz\nJ31dTk5OZF+bQ+68807i4+MZNmzYSffz9ddfk5aWBkB2djZTp0496WslSSooJYu6AEnS2fXCCy8w\nbdq04+7vlZeXR8mSR//fwaHgc9dddwHwf//v/2XRokV88cUXJ90HQLVq1Zg4cSJwMPhkZmZy8803\nn+50JEk6LS5uIElRrHfv3rz22mvUqlWLHj16sG7dOp5//nm6d+9OmTJl+Oyzz0hNTaVDhw48/PDD\nwMH9xz799FNatmzJqlWrqFGjBvfccw+jR49m7dq11KpVi5EjR/KrX/2KxMRE5syZw5133smyZcto\n165d5O5OhQoVyM3NJScnh3bt2rF48WL+67/+iz179lC9enV++ctf8pOf/OSocStWrFhkPy9J0vnn\nZBc38I6PJEWxjIwMPvzwQ2bMmMGUKVOOOPfll18yb948SpQowS233MKoUaNITU0lNzeXMmXK8Mwz\nzzB06NDIdbfddhvt2rUjOzs70se+ffs49Euq7t27/2AtF154IYMHDyYzM5Pnn38e4JjjSpJ0NviO\njyQVU506daJEiRIApKam8uijjzJixAh27Nhx3MfW/lPnzp3PqIbTHVeSpFNl8JGkYqp8+fKRzwMH\nDuSVV15hz549pKamsnr16lPuo2TJkhw4cACAAwcOsG/fvhNef7rjSpJ0qvzVmiSJdevWERcXR1xc\nHIsWLWL16tVcccUV7Nq166T7iImJISsrizvuuIPJkyezf//+o9pUrFjxiD6PNW7t2rULZE6SJB3O\nOz6SJIYPH05sbCzx8fGUKlWKm266ifj4eEqUKEFCQsJJLV/dq1cvZs2aRUJCAvPnzz/ibtAhN9xw\nAytXriQxMZHx48cfc1xJks4GV3WTJEmSdN462VXdvOMjSZIkKeoZfCRJkiRFPYOPpGJh5syZzJs3\nr6jLkCRJRcTgI6lYOJ3gk5eXd5aqkSRJhc3gI+mcl5OTQ+3atUlPT6dOnTqkpaWxe/duBg8eTHJy\nMrGxsdx///0cWqxlxIgR1K1bl/j4eLp06UJOTg4ZGRkMGzaMxMREZs+eTffu3Zk4cWJkjAoVKgAH\nA9J1111H+/btqVu3LgBvvfUWKSkpJCYm8sADD5Cfn1/4PwRJknRGDD6Szgtr1qyhT58+rFq1ikqV\nKvHCCy/Qt29fFi1axPLly9mzZw9TpkwB4JlnnuGzzz5j6dKlZGRkEBMTQ+/evXnkkUfIzs7muuuu\n+8GxFi9ezJ/+9Cc+//xzVq1axfjx45k7dy7Z2dmUKFGCsWPHFsaUJUlSATL4SDovXHHFFaSmpgLQ\ntWtX5syZw4wZM2jYsCFxcXFMnz6dFStWABAfH096ejpvvfUWJUue+j7NKSkp1KhRA4BPPvmErKws\nkpOTSUxM5JNPPmH9+vUFNzFJklQoDD5SMTdixAjq1KlD9erV6du37ylfn5OTQ2xs7Fmo7EhBEBz1\nvU+fPkycOJFly5bRq1cv9u7dC8Df/vY3/s//+T8sXryY5OTkY76rU7JkSQ4cOADAgQMH2LdvX+Tc\n4RtvhmHIPffcQ3Z2NtnZ2axZs4ZBgwadhRlKkqSzyeAjFXMvvPAC//jHP/jNb35T1KX8oA0bNjB/\n/nwAxo0bR5MmTQCoUqUKubm5kfd1Dhw4wMaNG7nhhhv4/e9/z86dO8nNzaVixYrs2rUr0l9MTAxZ\nWVkATJ48mf379x9z3BYtWjBx4kS+/fZbALZt28Y///nPszZPqTjZsWMHL7zwAgBff/01aWlpAGRn\nZzN16tRIu0GDBjF06NAiqVFS9DD4SMVY7969Wb9+PTfddBPbt2+PHM/JyaF58+bEx8fTokULNmzY\nAMA333zDbbfdRkJCAgkJCUetkrZ+/Xrq16/PokWLWLFiRWRBgPj4eNauXXtGtdaqVYtRo0ZRp04d\ntm/fzoMPPkivXr2IjY2ldevWJCcnA5Cfn0/Xrl2Ji4ujfv36PPTQQ1x00UXccsstTJo0KbK4Qa9e\nvZg1axYJCQnMnz//iLs8h6tbty5DhgyhVatWxMfH07JlSzZt2nRGc5F00OHBp1q1apFfYPxn8JGk\nghAcWgXpXJOUlBRmZmYWdRlS1IuJiSEzM5MpU6aQmZnJ888/zy233EJaWhr33HMPr732GpMnT+a9\n996jc+fONG7cmJ///Ofk5+eTm5vL9u3badeuHe+++y5dunRhzJgxJCQk0K9fPxo1akR6ejr79u0j\nPz+fsmXLnlaNOTk5tGvXjuXLlxfw7CUVpS5duvD+++9Tq1Ytrr76alatWsXixYv5r//6L/bs2UP1\n6tX55S9/yapVq9iwYQPr169nw4YN/PznP+ehhx4CDq66OGLECPbt20fDhg0jQeq+++4jMzOTIAjo\n0aMHjzzySFFOVdJZFARBVhiGSSdqd+pv/UqKevPnz+evf/0rAN26deMXv/gFANOnT+eNN94AoESJ\nElSuXJnt27ezefNmOnTowF//+tfIEtCNGzfmN7/5DV9++SUdO3bk6quvLprJSDpnPfPMMyxfvpzs\n7OzILzguvPBCBg8eHPlFDBx81G316tXMmDGDXbt2UatWLR588EG++OKLyKqLpUqVok+fPowdO5Z6\n9erx1VdfRX5ZsmPHjqKcpqRzhI+6STpjlStX5sorr2TOnDmRY3fddReTJ0+mbNmy3HzzzUyfPv20\n+4+JifFuj1TMtW3bltKlS1OlShV+/OMf88033xx31cWrrrqK9evX069fPz788EMqVapU1OVLOgcY\nfCQd5dprr+Uvf/kLAGPHjo3se9OiRQtefPFF4OC7NDt37gTgwgsvZNKkSbzxxhuMGzcOIPKXj4ce\neogOHTqwdOnSIpiJpGhRunTpyOcSJUqQl5d33FUXL774YpYsWUKzZs3IyMigZ8+eRVi5pHOFwUfS\nUUaOHMno0aOJj4/nzTff5E9/+hMAf/rTn5gxYwZxcXE0aNCAlStXRq4pX748U6ZMYdiwYUyePJl3\n3nmH2NhYEhMTWb58OXfffXdRTUfSOeo/V1s80fH/dLxVF7ds2cKBAwe4/fbbGTJkCIsXLy7w2iWd\nf1zcQJIkFZm77rqLpUuXUqdOHVatWsXy5cvZtm0brVu3Zv/+/ZHFDSpUqMBjjz0GQGxsLFOmTCEm\nJobx48fzu9/9jgMHDlCqVClGjRpF2bJluffeeyN7df3ud7/jpptuKsppSjqLTnZxA4OPJEmSpPPW\nyQYfH3WTJEmSFPUMPpIkSZKinsFHkiRJUtQz+EiSJEmKegYfSZIkSVHP4CNJkiQp6hl8JEmSJEU9\ng48kSZKkqGfwkSRJkhT1DD6SJEmSop7BR5IkSVLUM/hIkiRJinoGH0mSpDOUk5NDbGzsUcebNWtG\nZmbmGfc/ZswY+vbte8b9SMWZwUeSJElS1DP4SJIkFYC8vDzS09OpU6cOaWlp7N69+4jzb7/9NnFx\nccTGxjJgwIATHh89ejQ1a9YkJSWFuXPnFto8pGhl8JEkSSoAa9asoU+fPqxatYpKlSrxwgsvRM59\n/fXXDBgwgOnTp5Odnc2iRYt47733jnt806ZNPPXUU8ydO5c5c+awcuXKIpyZFB0MPpIkSQXgiiuu\nIDU1FYCuXbsyZ86cyLlFixbRrFkzqlatSsmSJUlPT+fTTz897vH//d//jRy/8MIL6dy5c1FNK2LH\njh2RMDdz5kzatWt3zHY9e/Y8paD23nvvGexUKAw+kiRJBSAIgh/8fr47PPj8kFdeeYW6desedTw/\nP/+Y7Q0+KiwGH0mSpAKwYcMG5s+fD8C4ceNo0qRJ5FxKSgqzZs1iy5Yt5Ofn8/bbb9O0adPjHm/Y\nsCGzZs1i69at7N+/nwkTJhTVtCIGDhzIunXrSExMpH///uTm5pKWlkbt2rVJT08nDEPgyJXsKlSo\nwH//93+TkJDA/PnzGThwIHXr1iU+Pp7HHnuMefPmMXnyZPr3709iYiLr1q0ryikqypUs6gIkSZKi\nQa1atRg1ahQ9evSgbt26PPjgg3zwwQcAXHbZZTzzzDPccMMNhGFI27Zt6dChA8Bxjw8aNIjGjRtz\n0UUXkZiYWGTzOuSZZ55h+fLlZGdnM3PmTDp06MCKFSuoVq0aqampzJ0794iwB/Ddd9/RsGFD/vCH\nP7B161buu+8+Vq9eTRAE7Nixg4suuoj27dvTrl070tLSimhmKi4MPpIkSWcoJiaG1atXH3V85syZ\nkc933nknd95551Ftjnf83nvv5d577y3QOgtSSkoKl19+OQCJiYnk5OQcFXxKlCjB7bffDkDlypUp\nU6YM9913H+3atTvuO0LS2eKjbpIkSTplpUuXjnwuUaIEeXl5R7UpU6YMJUqUAKBkyZIsXLiQtLQ0\npkyZQps2bQqtVgm84yNJkqSTULFiRXbt2nXa1+fm5rJ7925uvvlmUlNTueqqqwqkX+lkGXwkSZJ0\nQpdccgmpqanExsZStmxZLr300lO6fteuXXTo0IG9e/cShiF//OMfAejSpQu9evVixIgRTJw4kZ/9\n7Gdno3yJ4NAKHOeapKSk8NCKIJIkSZJ0LEEQZIVhmHSidr7jI0mSJCnqGXwkSZIkRT2DjyRJkqSo\nZ/CRJEmSFPUMPtJJuPbaa4u6BEmSJJ0Bg490EubNm1fUJUiSJOkMGHykk1ChQgUAZs6cSdOmTenQ\noQNXXXUVAwcOZOzYsaSkpBAXF8e6desA+OCDD2jYsCH169fnxhtv5JtvvgFg8+bNtGzZknr16tGz\nZ09++tOfsmXLFgDeeustUlJSSExM5IEHHiA/P5/8/Hy6d+9ObGwscXFxDBs2rGh+AJIkSec5g490\nipYsWUJGRgarVq3izTff5PPPP2fhwoX07NmTkSNHAtCkSRMWLFjAZ599RpcuXXj22WcBePrpp2ne\nvDkrVqwgLS2NDRs2ALBq1SrGjx/P3Llzyc7OpkSJEowdO5bs7Gy++uorli9fzrJly7j33nuLbN6S\nJEnns5JFXYB0vklOTuayyy4D4Gc/+xmtWrUCIC4ujhkzZgDw5Zdf0rlzZzZt2sS+ffuoUaMGAHPm\nzGHSpEkAtGnThosvvhiATz75hKysLJKTkwHYs2cPP/7xj7nllltYv349/fr1o23btpGxJEmSdGq8\n46P/n707j8uqzP8//jogiWmipZk6NtKMC/sOIYG4JKamZZIZLmhYadq0mTZlkpE5SU0/adEcE9fG\n1HJpNVJHSI1NcFcwb82kXBITFBU4vz/M+xupqbHcLO/n48HDc859Xed8rpvM+3Nfm1yj+vXrW4/t\n7Oys53Z2dhQXFwMwduxYxowZw9atW5k5cyZFRUV/eE/TNBk2bBhZWVlkZWWxe/duYmNjadq0KdnZ\n2YSHhzNjxgxiYmIqr2EiIiIitZgSH5FKcOLECVq3bg3A3LlzrddDQkL48MMPAVi9ejXHjx8HoFu3\nbixdupTDhw8D8PPPP7N//36OHj1KaWkp9913H3FxcWRmZlZxS0RERERqBw11E6kEsbGxREZG0rRp\nU7p27cq+ffsAmDRpEoMGDWL+/PkEBwdzyy23cMMNN9CsWTPi4uLo0aMHpaWlODg48Pbbb9OgQQOG\nDx9OaWkpAK+++qotmyUiIiJSYxmmado6hkvy9/c309PTbR2GSIU6c+YM9vb21KtXj40bNzJq1Ciy\nsrJsHZaIiIhIjWUYRoZpmv5XKqceH5EqdODAAe6//35KS0u57rrrmDVrlq1DEhEREakTlPiIVKF2\n7dqxefNmW4chIiIiUudocQMREbmi6dOn4+LiQlRU1FXX6dWrF/n5+eTn5/POO+9UYnQiIiJXpjk+\nIiJyRR07diQpKYm//OUv1mvFxcXUq3flgQMWi4U+ffqwbdu2ygxRRETqqKud46MeHxER+UOPPvoo\n3333HXfddRdOTk4MGTKEkJAQhgwZQmJiImPGjLGW7dOnD+vWrQOgbdu2HD16lAkTJrB37168vb0Z\nN24ceXl5hIWF4e3tjbu7O8nJyTZqmYiI1CWa4yMiIn9oxowZfPHFF6xdu5a33nqLVatWkZKSQoMG\nDUhMTLxi/alTp7Jt2zbrCoavv/46ERERPP/885SUlHDq1KlKboGIiIgSHxERuUZ9+/alQYMGf7p+\nQEAAI0aM4Ny5c9xzzz14e3tXYHQiIiKXpqFuIiJyTRo2bGg9rlevnnWDXYCioqIr1g8LC2P9+vW0\nbt2a6Oho5s2bVylxioiI/JYSHxER+dPatm1LVlYWpaWlfP/996Smpl5U5oYbbuDkyZPW8/3799Oi\nRQtGjhxJTEwMmZmZVRmyiIjUURrqJiIif1pISAjOzs64urri4uKCr6/vRWVuuukmQkJCcHd35667\n7sLd3Z1p06bh4OBAo0aN1OMjIiJVQstZi4iIiIhIjVWly1kbhtHTMIzdhmHkGoYx4RKv1zcMY/Gv\nr39rGEbbiniuiIiIiIjI1Sh34mMYhj3wNnAX4AoMMgzD9XfFHgKOm6b5d+DfwL/K+1wREREREZGr\nVRE9PoFArmma35mmeRb4L9Dvd2X6AXN/PV4KdDMMw6iAZ4uIiIiIiFxRRSQ+rYHvf3N+8Ndrlyxj\nmmYxcAK46fc3MgzjYcMw0g3DSD9y5EgFhCYiIiIiIlLNlrM2TfM90zT9TdP0b968ua3DERERERGR\nWqIiEp8fgDa/Of/Lr9cuWcYwjHqAE3CsAp4tIiIiIiJyRRWR+KQB7QzDcDYM4zrgAWDl78qsBIb9\nejwAWGNW13W0RURERESk1in3BqamaRYbhjEG+BKwB943TXO7YRiTgXTTNFcCs4H5hmHkAj9zPjkS\nERERERGpEuVOfABM0/wM+Ox31178zXEREFkRzxIREREREblW1WpxAxERERERkcqgxEdERERERGo9\nJT4iIiIiInJFFouFjh07Eh0dTfv27YmKiiIpKYmQkBDatWtHamoqP//8M/fccw+enp7cfvvtbNmy\nBYDY2FhGjBhBeHg4t912G9OnT7fed8GCBQQGBuLt7c0jjzxCSUkJ77//Pk888YS1zKxZs3jyySfL\nFb8SH5FK0rZtW44ePWrrMEREREQqTG5uLk8//TS7du1i165dLFq0iJSUFOLj45kyZQqTJk3Cx8eH\nLVu2MGXKFIYOHWqtu2vXLr788ktSU1N56aWXOHfuHDt37mTx4sV88803ZGVlYW9vz8KFC7n//vtZ\ntXDCfcQAACAASURBVGoV586dA2DOnDmMGDGiXLFXyOIGIiIiIiJS+zk7O+Ph4QGAm5sb3bp1wzAM\nPDw8sFgs7N+/n2XLlgHQtWtXjh07xi+//AJA7969qV+/PvXr1+fmm2/mp59+4uuvvyYjI4OAgAAA\nTp8+zc0330yjRo3o2rUrn3zyCS4uLpw7d8763D9LPT4iFaCwsJDevXvj5eWFu7s7ixcvBiAhIQFf\nX188PDzYtWuXteyIESMIDAzEx8eHFStWAFBSUsK4ceMICAjA09OTmTNnArBu3TrCwsLo3bs3HTp0\n4NFHH6W0tNQ2DRUREZE6rX79+tZjOzs767mdnR3FxcVXXdfe3p7i4mJM02TYsGFkZWWRlZXF7t27\niY2NBSAmJobExETmzJnD8OHDyx27Eh+RCvDFF1/QqlUrsrOz2bZtGz179gSgWbNmZGZmMmrUKOLj\n4wF45ZVX6Nq1K6mpqaxdu5Zx48ZRWFjI7NmzcXJyIi0tjbS0NGbNmsW+ffsASE1NJSEhgR07drB3\n714++ugjm7VVRERE5HJCQ0NZuHAhcP7L22bNmtG4cePLlu/WrRtLly7l8OHDAPz888/s378fgKCg\nIL7//nsWLVrEoEGDyh2bEh+RCuDh4cFXX33F+PHjSU5OxsnJCYD+/fsD4Ofnh8ViAWD16tVMnToV\nb29vwsPDKSoq4sCBA6xevZp58+bh7e1NUFAQx44dIycnB4DAwEBuu+027O3tGTRoECkpKTZpp1yb\nK33zJSIiUtvExsaSkZGBp6cnEyZMYO7cuX9Y3tXVlbi4OHr06IGnpyd33nkneXl51tfvv/9+QkJC\naNq0ablj0xwfkQrQvn17MjMz+eyzz3jhhRfo1q0b8H9duhe6cwFM02TZsmV06NChzD1M0yQhIYGI\niIgy19etW4dhGGWu/f5cbOPll19mwYIFNG/enDZt2uDn58cnn3yCt7c3KSkpDBo0iPbt2xMXF8fZ\ns2e56aabWLhwIS1atCA2Npa9e/eSm5vL0aNHefbZZxk5ciQA06ZN48MPP+TMmTPce++9vPTSSxQW\nFnL//fdz8OBBSkpKmDhxIgMHDrTxOyAiInVJ27Zt2bZtm/U8MTHxkq8tX778oroXhq9d8Nv7DBw4\n8LL/pqWkpJR7NbcLlPiIVIBDhw5x4403MnjwYJo0acJ//vOfy5aNiIggISGBhIQEDMNg8+bN+Pj4\nEBERwbvvvkvXrl1xcHBgz549tG7dGjg/1G3fvn389a9/ZfHixTz88MNV1TS5jLS0NJYtW0Z2djbn\nzp3D19cXPz8/AM6ePUt6ejoAx48fZ9OmTRiGwX/+8x9ee+01Xn/9dQC2bNnCpk2bKCwsxMfHh969\ne7Nt2zZycnJITU3FNE369u3L+vXrOXLkCK1ateLTTz8F4MSJE7ZpuIiISBXIz88nMDAQLy8v6xfK\n5aXER6QCbN26lXHjxmFnZ4eDgwPvvvsuAwYMuGTZiRMn8sQTT+Dp6UlpaSnOzs588sknxMTEYLFY\n8PX1xTRNmjdvbv3GJCAggDFjxpCbm0uXLl249957q7J5cgnffPMN/fr1w9HREUdHR+6++27ra7/9\n1urgwYMMHDiQvLw8zp49i7Ozs/W1fv360aBBAxo0aECXLl1ITU0lJSWF1atX4+PjA0BBQQE5OTmE\nhoby9NNPM378ePr06UNoaGjVNVZERKSKNWnShD179lToPZX4iFSAiIiIi4aoXZjTA+Dv78+6desA\naNCggXXFtt+ys7NjypQpTJky5aLXGjduzCeffFKhMUvladiwofV47NixPPXUU/Tt25d169aV6eq/\n1BBG0zR57rnneOSRRy667++HU7744ouV1gYREZHaRosbiIj8CSEhIaxatYqioiIKCgoum5ieOHHC\nOmTx9xM8V6xYQVFREceOHWPdunUEBAQQERHB+++/T0FBAQA//PADhw8f5tChQ1x//fUMHjyYcePG\nkZmZWbkNFBERqWXU4yNSzYWHhxMeHm7rMOR3AgIC6Nu3L56enrRo0QIPDw/ran6/FRsbS2RkJE2b\nNqVr167WJcoBPD096dKlC0ePHmXixIm0atWKVq1asXPnToKDgwFo1KgRCxYsIDc396LhlCIiInL1\nDNM0bR3DJfn7+5sXJgeLiFRHBQUFNGrUiFOnThEWFsZ7772Hr6/vVdWNjY2lUaNGPPPMM5UcpYiI\nSO1mGEaGaZr+VyqnHh8RkT/p4YcfZseOHRQVFTFs2LCrTnpERESk6qnHR0REREREaqyr7fHR4gYi\nIiJ1RGxsLPHx8QBER0ezdOlSG0ckIlJ1lPiIiIiIiEitpzk+UuEsFgt9+vRh27Zttg5FRKTWe/nl\nl1mwYAHNmzenTZs2+Pn54eTkxHvvvcfZs2f5+9//zvz587n++uttHaqIiE2px0eqleLiYluHICJS\nY6SlpbFs2TKys7P5/PPPuTA3tn///qSlpZGdnY2LiwuzZ8+2caQiIranHh+pFMXFxURFRZGZmYmb\nmxvz5s0jPj6eVatWcfr0aTp16sTMmTMxDIPw8HC8vb1JSUlh0KBB3Hrrrbz00kvY29vj5OTE+vXr\nbd0cEZFq6ZtvvqFfv344Ojri6OjI3XffDcC2bdt44YUXyM/Pp6CggIiICBtHKiJie+rxkUqxe/du\nRo8ezc6dO2ncuDHvvPMOY8aMIS0tjW3btnH69OkyO92fPXuW9PR0nn76aSZPnsyXX35JdnY2K1eu\ntGEr5FodOnSIAQMG2DoM4PyQS3d3d1uHIWIT0dHRvPXWW2zdupVJkyZRVFRk65BERGxOiY9UijZt\n2hASEgLA4MGDSUlJYe3atQQFBeHh4cGaNWvYvn27tfzAgQOtxyEhIURHRzNr1ixKSkqqPHb581q1\namWzVaL034rURSEhIaxatYqioiIKCgqsXyidPHmSli1bcu7cORYuXGjjKEVEqgclPlIpDMO46Hz0\n6NEsXbqUrVu3MnLkyDLfQDZs2NB6PGPGDOLi4vj+++/x8/Pj2LFjVRa3XL0JEybw9ttvW88vLJN7\noZclMTGR/v3707NnT9q1a8ezzz5rLbt69WqCg4Px9fUlMjKSgoIC1qxZwz333GMt89VXX3HvvfcC\nMGrUKPz9/XFzc2PSpEnWMm3btmX8+PH4+vqyZMkSMjIy8PLywsvLq0xs27dvJzAwEG9vbzw9PcnJ\nyam090WkKgUEBNC3b188PT2566678PDwwMnJiZdffpmgoCBCQkLo2LGjrcMUEakeTNOslj9+fn6m\n1Ez79u0zAXPDhg2maZrmQw89ZMbHx5s333yzeerUKfPkyZOmm5ubOWnSJNM0TbNz585mWlqatX5u\nbq712N/f39y8eXOVxi9XJzMz0wwLC7Oeu7i4mOvXrzfd3NxM0zTNOXPmmM7OzmZ+fr55+vRp89Zb\nbzUPHDhgHjlyxAwNDTULCgpM0zTNqVOnmi+99JJZWlpqdujQwTx8+LBpmqY5aNAgc+XKlaZpmuax\nY8dM0zTN4uJis3PnzmZ2drZpmqb517/+1fzXv/5ljcHDw8P83//+Z5qmaT7zzDPWWMaMGWMuWLDA\nNE3TPHPmjHnq1KlKe19EqtrJkydN0zTNwsJC08/Pz8zIyLBxRCIiVQtIN68iv9DiBlIpOnTowNtv\nv82IESNwdXVl1KhRHD9+HHd3d2655RYCAgIuW3fcuHHk5ORgmibdunXDy8urCiOXq+Xj48Phw4c5\ndOgQR44coWnTprRp06ZMmW7duuHk5ASAq6sr+/fvJz8/nx07dliHQp49e5bg4GAMw2DIkCEsWLCA\n4cOHs3HjRubNmwfAhx9+yHvvvUdxcTF5eXns2LEDT09P4P+GSebn55Ofn09YWBgAQ4YM4fPPPwcg\nODiYV155hYMHD9K/f3/atWtX+W+QSBV5+OGH2bFjB0VFRQwbNgxfX19bhyQiUi0p8ZEK17ZtW3bt\n2nXR9bi4OOLi4i66vm7dujLnH330UWWFJhUsMjKSpUuX8uOPP5aZp3VB/fr1rcf29vYUFxdjmiZ3\n3nknH3zwwUXlhw8fzt13342joyORkZHUq1ePffv2ER8fT1paGk2bNiU6OvqywyQv58EHHyQoKIhP\nP/2UXr16MXPmTLp27fonWy1SvSxatMjWIYiI1Aia4yMif9rAgQP573//y9KlS4mMjLyqOrfffjvf\nfPMNubm5ABQWFrJnzx7g/OIIrVq1Ii4ujuHDhwPwyy+/0LBhQ5ycnPjpp5+svTi/16RJE5o0aUJK\nSgpAmQnd3333HbfddhuPP/44/fr1Y8uWLX+6zSIiIlIzKfERkT/Nzc2NkydP0rp1a1q2bHlVdZo3\nb05iYiKDBg3C09OT4ODgMj2EUVFRtGnTBhcXFwC8vLzw8fGhY8eOPPjgg9YhcpcyZ84cHnvsMby9\nvTk/5Pe8Dz/8EHd3d7y9vdm2bRtDhw79ky0WERGRmsr47YeD6sTf39+8sAO1iNQdY8aMwcfHh4ce\nesjWoYiIiEgNYBhGhmma/lcqpzk+IlJt+Pn50bBhQ15//XVbhyIiIiK1jBIfEak2MjIybB2CiIiI\n1FKa4yMiIiIiIrWeEh8REREREan1lPiIiIiIiEitp8RHRERERERqPSU+IlJrHDp0iAEDBtg6DBER\nEamGlPiISK3RqlUrli5dauswREREpBpS4iMiNdKECRN4++23reexsbHEx8fj7u4OQGJiIv3796dn\nz560a9eOZ5991lp29erVBAcH4+vrS2RkJAUFBdZ7urq64unpyTPPPFO1DRIREZFKpcRHRGqkgQMH\n8uGHH1rPP/zwQ4KCgsqUycrKYvHixWzdupXFixfz/fffc/ToUeLi4khKSiIzMxN/f3/eeOMNjh07\nxscff8z27dvZsmULL7zwQlU3SURERCqRNjAVkRrJx8eHw4cPc+jQIY4cOULTpk1p06ZNmTLdunXD\nyckJAFdXV/bv309+fj47duwgJCQEgLNnzxIcHIyTkxOOjo489NBD9OnThz59+lR5m0RERKTyKPER\nkRorMjKSpUuX8uOPPzJw4MCLXq9fv7712N7enuLiYkzT5M477+SDDz64qHxqaipff/01S5cu5a23\n3mLNmjWVGr+IiIhUHSU+IlJjDRw4kJEjR3L06FH+97//cebMmSvWuf3223nsscfIzc3l73//O4WF\nhfzwww+0atWKU6dO0atXL0JCQrjtttuqoAUiIiJSVZT4iEiN5ebmxsmTJ2ndujUtW7bEYrFcsU7z\n5s1JTExk0KBB1kQpLi6OG264gX79+lFUVIRpmrzxxhuVHL2IiIhUJcM0TVvHcEn+/v5menq6rcMQ\nEREREZFqzDCMDNM0/a9UTqu61RLFxcW2DkFEREREpNpS4vMnWSwWOnbsSHR0NO3btycqKoqkpCRC\nQkJo164dqampFBYWMmLECAIDA/Hx8WHFihXWuqGhofj6+uLr68uGDRsAyMvLIywsDG9vb9zd3UlO\nTgagUaNG1ucuXbqU6OhoAKKjo3n00UcJCgri2Wefvezztm/fTmBgIN7e3nh6epKTk1OF75SIiIiI\niO1pjk855ObmsmTJEt5//30CAgJYtGgRKSkprFy5kilTpuDq6krXrl15//33yc/PJzAwkO7du3Pz\nzTfz1Vdf4ejoSE5ODoMGDSI9PZ1FixYRERHB888/T0lJCadOnbpiDAcPHmTDhg3Y29vzz3/+85LP\nmzFjBv/4xz+Iiori7NmzlJSUVMG7IyIiIiJSfSjxKQdnZ2c8PDyA85Osu3XrhmEYeHh4YLFYOHjw\nICtXriQ+Ph6AoqIiDhw4QKtWrRgzZgxZWVnY29uzZ88eAAICAhgxYgTnzp3jnnvuwdvb+4oxREZG\nYm9vD5zfjf5SzwsODuaVV17h4MGD9O/fn3bt2lXG21FnhIeHEx8fj7//FYeSioiIiEg1ocSnHH67\nR4idnZ313M7OjuLiYuzt7Vm2bBkdOnQoUy82NpYWLVqQnZ1NaWkpjo6OAISFhbF+/Xo+/fRToqOj\neeqppxg6dCiGYVjrFhUVlblXw4YNrcemaV7yeS4uLgQFBfHpp5/Sq1cvZs6cSdeuXSvmTRARERER\nqQE0x6cSRUREkJCQwIWV8zZv3gzAiRMnaNmyJXZ2dsyfP9869Gz//v20aNGCkSNHEhMTQ2ZmJgAt\nWrRg586dlJaW8vHHH1/z87777jtuu+02Hn/8cfr168eWLVsqrc012T333IOfnx9ubm689957lJSU\nEB0djbu7Ox4eHvz73/+2ll2yZAmBgYG0b9/eOherpKSEcePGERAQgKenJzNnzgQuP3dLRERERKqO\nenwq0cSJE3niiSfw9PSktLQUZ2dnPvnkE0aPHs19993HvHnz6Nmzp7XXZt26dUybNg0HBwcaNWrE\nvHnzAJg6dSp9+vShefPm+Pv7U1BQcE3P+/DDD5k/fz4ODg7ccsst/POf/6yy96Amef/997nxxhs5\nffo0AQEB+Pn58cMPP7Bt2zYA8vPzrWWLi4tJTU3ls88+46WXXiIpKYnZs2fj5OREWloaZ86cISQk\nhB49evDRRx9d89wtEREREalY2sdH5FexsbHWHjWLxcIXX3zB4MGD6dWrF71796ZHjx7Y2dkRHh7O\nK6+8QkhICD/99BMhISHk5uYyYMAAtmzZwvXXXw+c79mbOXMmjo6OjBgxgsGDB1/13C0RERERuTra\nx0fkGqxbt46kpCQ2btxIdnY2Pj4+nDlzhuzsbMLDw5kxYwYxMTHW8hfmc9nb21v3UDJNk4SEBLKy\nssjKymLfvn306NHDOnerdevWREdHW3vyRERERKTqKPER4XzvTNOmTbn++uvZtWsXmzZt4ujRo5SW\nlnLfffcRFxdnnXN1OREREbz77rucO3cOgD179lBYWHjZuVsiIiIiUnU0x0cE6NmzJzNmzMDFxYUO\nHTpw++2388MPPxAeHk5paSkAr7766h/eIyYmBovFgq+vL6Zp0rx5c5YvX37ZuVsiIiIiUnU0x0dE\nRERERGoszfERERERqSAWiwV3d/dKf07btm05evRopT9HpC5S4iMiIiJSAS7syyci1ZMSHxEREZGr\nUFxcTFRUFC4uLgwYMIBTp07Rtm1bxo8fj6+vL0uWLGHv3r307NkTPz8/QkND2bVrFwCrVq0iKCgI\nHx8funfvzk8//QTAsWPH6NGjB25ubsTExFBdpyCI1AZKfERERESuwu7duxk9ejQ7d+6kcePGvPPO\nOwDcdNNNZGZm8sADD/Dwww+TkJBARkYG8fHxjB49GoA77riDTZs2sXnzZh544AFee+01AF566SXu\nuOMOtm/fzr333suBAwds1j6R2k6ruomIiIhchTZt2hASEgLA4MGDmT59OgADBw4EoKCggA0bNhAZ\nGWmtc+bMGQAOHjzIwIEDycvL4+zZszg7OwOwfv16PvroIwB69+5N06ZNq6w9InWNEh8RERGRq2AY\nxiXPGzZsCEBpaSlNmjQhKyvrorpjx47lqaeeom/fvqxbt47Y2NhKj1dEytJQNxEREZGrcODAATZu\n3AjAokWLuOOOO8q83rhxY5ydnVmyZAkApmmSnZ0NnN8ou3Xr1gDMnTvXWicsLIxFixYB8Pnnn3P8\n+PFKb4dIXaXER0REROQqdOjQgbfffhsXFxeOHz/OqFGjLiqzcOFCZs+ejZeXF25ubqxYsQKA2NhY\nIiMj8fPzo1mzZtbykyZNYv369bi5ufHRRx9x6623Vll7ROoabWAqIiIiIiI1ljYwFRERERER+ZUS\nHxERKTftNi8iItWdEh8REREREan1lPiIiNRB06ZNs+5B8uSTT9K1a1cA1qxZQ1RUFB988AEeHh64\nu7szfvx4a73LXRcREanulPiIiNRBoaGhJCcnA5Cenk5BQQHnzp0jOTmZ9u3bM378eNasWUNWVhZp\naWksX76cQ4cOXfK6iMif8eKLL5KUlGTrMKQOUeIjIlIH+fn5kZGRwS+//EL9+vUJDg4mPT2d5ORk\nmjRpQnh4OM2bN6devXpERUWxfv160tLSLnldRORalZSUMHnyZLp3727rUKQOUeIjIlIHOTg44Ozs\nTGJiIp06dSI0NJS1a9eSm5tL27ZtbR2eiNRgFouFjh07EhUVhYuLCwMGDODUqVO0bduW8ePH4+vr\ny5IlS4iOjmbp0qUApKWl0alTJ7y8vAgMDOTkyZOUlJQwbtw4AgIC8PT0ZObMmTZumdR0SnxEROqo\n0NBQ4uPjCQsLIzQ0lBkzZuDj40NgYCD/+9//OHr0KCUlJXzwwQd07tz5stdFRH5v9+7djB49mp07\nd9K4cWPeeecdAG666SYyMzN54IEHrGXPnj3LwIED+X//7/+RnZ1NUlISDRo0YPbs2Tg5OZGWlkZa\nWhqzZs1i3759tmqS1AJKfERE6qjQ0FDy8vIIDg6mRYsWODo6EhoaSsuWLZk6dSpdunTBy8sLPz8/\n+vXrd9nrUn7Tp0/HxcWF1q1bM2bMGFuHI1Xowu8+Kirqquv06tWL/Px88vPzrQlFddOmTRtCQkIA\nGDx4MCkpKQAMHDjworK7d++mZcuWBAQEANC4cWPq1avH6tWrmTdvHt7e3gQFBXHs2DFycnKqrhFS\n69SzdQAiImIb3bp149y5c9bzPXv2WI8HDRrEoEGDLqpzuesWi6VSYqwr3nnnHZKSkkhKSiI9Pd3W\n4UgVuvC7/8tf/mK9VlxcTL16l/+I9tlnnwHn/9698847jB49utLjvFaGYVzyvGHDhld9D9M0SUhI\nICIiokJjk7pLPT4iIiI29Oijj/Ldd99x1113cfz4cev1VatWERQUhI+PD927d+enn34C4B//+AeT\nJ08G4MsvvyQsLIzS0lKbxC7l89vfvZOTE0OGDCEkJIQhQ4aQmJhYpvevT58+rFu3Dvi/DYMnTJjA\n3r178fb2Zty4cTZqxaUdOHCAjRs3ArBo0SLuuOOOy5bt0KEDeXl5pKWlAXDy5EmKi4uJiIjg3Xff\ntX5Bs2fPHgoLCys/eKm1lPiIiIjY0IwZM2jVqhVr166ladOm1ut33HEHmzZtYvPmzTzwwAO89tpr\nALz66qssXryYtWvX8vjjjzNnzhzs7PTPeU3029/9k08+yY4dO0hKSuKDDz64qvpTp07lb3/7G1lZ\nWUybNq2So702HTp04O2338bFxYXjx48zatSoy5a97rrrWLx4MWPHjsXLy4s777yToqIiYmJicHV1\nxdfXF3d3dx555BGKi4ursBVS22iom4iISDV08OBBBg4cSF5eHmfPnsXZ2RmA66+/nlmzZhEWFsa/\n//1v/va3v9k4Uqkoffv2pUGDBrYOo0LUq1ePBQsWlLn2+yGxiYmJ1uOAgAA2bdp00X2mTJnClClT\nKiNEqYP0FZGIiEg1NHbsWMaMGcPWrVuZOXMmRUVF1te2bt3KTTfdxKFDh2wYoVS0385/qVevXpkh\njL/9/YvIn6PER0REpBo6ceIErVu3BmDu3LnW6/v37+f1119n8+bNfP7553z77be2ClEqUdu2bcnK\nyqK0tJTvv/+e1NTUi8rccMMNnDx50gbR/bG2bduybds2W4chchElPiJSazVq1KhC7pOVlWVdRUmk\nqsTGxhIZGYmfnx/NmjUDzq9y9dBDDxEfH0+rVq2YPXs2MTEx6g2ohUJCQnB2dsbV1ZXHH38cX1/f\ni8rcdNNNhISE4O7uXu0WNxCpjgzTNP98ZcO4EVgMtAUswP2maR7/XRlv4F2gMVACvGKa5uIr3dvf\n39/Ukp4iUh6NGjWioKCg3PdJTEwkPT2dt956qwKiEhERkYpkGEaGaZr+VypX3h6fCcDXpmm2A77+\n9fz3TgFDTdN0A3oCbxqG0aSczxURYdq0aUyfPh2AJ598kq5duwKwZs0a62aAzz//PF5eXtx+++3W\n5YCPHDnCfffdR0BAAAEBAXzzzTcApKamEhwcjI+PD506dWL37t2cPXuWF198kcWLF+Pt7c3ixVf8\n3kZERESqofImPv2ACwOP5wL3/L6AaZp7TNPM+fX4EHAYaF7O50oNUlHDjWJjY4mPj6+Qe0ntEBoa\nSnJyMgDp6ekUFBRw7tw5kpOTCQsLo7CwkNtvv53s7GzCwsKYNWsWcH4flCeffJK0tDSWLVtGTEwM\nAB07diQ5OZnNmzczefJk/vnPf3LdddcxefJkBg4cSFZW1iV3HRcREZHqr7zLWbcwTTPv1+MfgRZ/\nVNgwjEDgOmBvOZ8rIoKfnx8ZGRn88ssv1K9fH19fX9LT00lOTmb69Olcd9119OnTx1r2q6++AiAp\nKYkdO3ZY7/PLL79QUFDAiRMnGDZsGDk5ORiGYd00T0RERGq+K/b4GIaRZBjGtkv89PttOfP8ZKHL\nThgyDKMlMB8YbprmJbeYNgzjYcMw0g3DSD9y5Mg1NkWqu4KCArp164avry8eHh6sWLECOL+uf8eO\nHYmOjqZ9+/ZERUWRlJRESEgI7dq1K7OSTXZ2NsHBwbRr18767X1eXh5hYWF4e3vj7u5u7QGQ2s/B\nwQFnZ2cSExPp1KkToaGhrF27ltzcXFxcXHBwcMAwDADs7e2tG9+VlpayadMmsrKyyMrK4ocffqBR\no0ZMnDiRLl26sG3bNlatWqUJ4yIiIrXIFRMf0zS7m6bpfomfFcBPvyY0FxKbw5e6h2EYjYFPgedN\n07x4d6r/e9Z7pmn6m6bp37y5RsPVNo6Ojnz88cdkZmaydu1ann76aS4srpGbm8vTTz/Nrl272LVr\nF4sWLSIlJYX4+PgyG5dt2bKFNWvWsHHjRiZPnsyhQ4dYtGgRERERZGVlkZ2djbe3t62aKDYQGhpK\nfHw8YWFhhIaGMmPGDHx8fKwJz6X06NGDhIQE63lWVhZQdvng326sV12XjBUREZGrV945PiuBYb8e\nDwNW/L6AYRjXAR8D80zTXFrO50kNZpom//znP/H09KR79+788MMP1snmzs7OeHh4YGdnh5ubBbEx\nZgAAHWJJREFUG926dcMwDDw8PMrs9NyvXz8aNGhAs2bN6NKlC6mpqQQEBDBnzhxiY2PZunUrN9xw\ng41aKLYQGhpKXl4ewcHBtGjRAkdHR0JDQ/+wzvTp00lPT8fT0xNXV1dmzJgBwLPPPstzzz2Hj4+P\ntXcIoEuXLuzYsUOLG4iIiNRg5Z3jMxX40DCMh4D9wP0AhmH4A4+aphnz67Uw4CbDMKJ/rRdtmmZW\nOZ8tNczChQs5cuQIGRkZODg40LZtW+tQovr161vL2dnZWc/t7OzKfAD9/bf4hmEQFhbG+vXr+fTT\nT4mOjuapp55i6NChVdAiqQ66detWZi7Onj17rMe/Xcp6wIABDBgwAIBmzZpdMoEJDg4uUz8uLg6A\nG2+8kbS0tAqPXURERKpOuXp8TNM8ZppmN9M02/06JO7nX6+n/5r0YJrmAtM0HUzT9P7Nj5KeOujE\niRPcfPPNODg4sHbtWvbv33/N91ixYgVFRUUcO3aMdevWERAQwP79+2nRogUjR44kJiaGzMzMSohe\nRERERGqy8vb4iFy1qKgo7r77bjw8PPD396djx47XfA9PT0+6dOnC0aNHmThxIq1atWLu3LlMmzYN\nBwcHGjVqxLx58yohehERERGpyYwLk8urG39/fzM9Pd3WYYiIiIiISDVmGEaGaZr+VypX3sUNpBZ6\n4403cHd3x93dnTfffJO0tDQ8PT0pKiqisLAQNzc3tm3bBsC0adMICAjA09OTSZMmAeeXp3ZxcWHk\nyJG4ubnRo0cPTp8+bcsmiYiIiEgdp8RHysjIyGDOnDl8++23bNq0iVmzZlGvXj369u3LCy+8wLPP\nPsvgwYNxd3dn9erV5OTkkJqaSlZWFhkZGaxfvx6AnJwcHnvsMbZv306TJk1YtmyZjVsmIiIiInWZ\n5vhIGSkpKdx77700bNgQgP79+5OcnMyLL75IQEAAjo6OTJ8+HYDVq1ezevVqfHx8gPMraOXk5HDr\nrbfi7Oxs3U/Hz8+vzJLUIiIiIiJVTYmPXJVjx45RUFDAuXPnKCoqomHDhpimyXPPPccjjzxSpqzF\nYimzPLW9vb2GuomIiIiITWmom5QRGhrK8uXLOXXqFIWFhXz88ceEhobyyCOP8PLLLxMVFcX48eMB\niIiI4P3337fulfLDDz9w+PBhW4YvIiIiInJJ6vGRMnx9fYmOjiYwMBCAmJgYtm7dioODAw8++CAl\nJSV06tSJNWvW0KNHD3bu3ElwcDAAjRo1YsGCBdjb29uyCSIiIiIiF9Fy1iIidVxxcTH16ul7MBER\nqZmudjlr/UsnIlKLzJs3j/j4eAzDwNPTk5dffpkRI0Zw9OhRmjdvzpw5c7j11luJjo7G0dGRzZs3\nExISQuPGjTlw4ADfffcdBw4c4IknnuDxxx+3dXNEREQqjBIfEZFaYvv27cTFxbFhwwaaNWvGzz//\nzLBhw6w/77//Po8//jjLly8H4ODBg2zYsAF7e3tiY2PZtWsXa9eu5eTJk3To0IFRo0bh4OBg41aJ\niIhUDC1uICJSS6xZs4bIyEiaNWsGwI033sjGjRt58MEHARgyZAgpKSnW8pGRkWXm5PXu3Zv69evT\nrFkzbr75Zn766aeqbYCIiEglUuIjIlJHXdiv64LfL0NfXFxc1SGJiIhUGiU+IiK1RNeuXVmyZAnH\njh0D4Oeff6ZTp07897//BWDhwoWEhobaMkQRERGb0RwfEZFaws3Njeeff57OnTtjb2+Pj48PCQkJ\nDB8+nGnTplkXNxAREamLtJy1iIiIiIjUWFe7nLWGuomIiIiISK2nxEdERERERGo9JT4iIiIiIlLr\nKfEREREREZFaT4mPiIiIiIjUekp8RERERESk1lPiIyIiIiIitZ4SHxEREREbslgsLFq0yNZhiNR6\nSnxEREREbEiJj0jVUOIjIiIicpUsFgsdO3YkOjqa9u3bExUVRVJSEiEhIbRr147U1FQKCwsZMWIE\ngYGB+Pj4sGLFCmvd0NBQfH198fX1ZcOGDQBMmDCB5ORkvL29+fe//23L5onUaoZpmraO4ZL8/f3N\n9PR0W4chIiIiYmWxWPj73//O5s2bcXNzIyAgAC8vL2bPns3KlSuZM2cOrq6uuLq6MnjwYPLz8wkM\nDGTz5s0YhoGdnR2Ojo7k5OQwaNAg0tPTWbduHfHx8XzyySe2bp5IjWQYRoZpmv5XKlevKoIRERER\nqS2cnZ3x8PAAwM3NjW7dumEYBh4eHlgsFg4ePMjKlSuJj48HoKioiAMHDtCqVSvGjBlDVlYW9vb2\n7Nmzx5bNEKlzlPiIiIiIXIP69etbj+3s7KzndnZ2FBcXY29vz7Jly+jQoUOZerGxsbRo0YLs7GxK\nS0txdHSs0rhF6jrN8RERERGpQBERESQkJHBhOsHmzZsBOHHiBC1btsTOzo758+dTUlICwA033MDJ\nkydtFq9IXaHER0RERKQCTZw4kXPnzuHp6YmbmxsTJ04EYPTo0cydOxcvLy927dpFw4YNAfD09MTe\n3h4vLy8tbiBSibS4gYiISDXVq1cvFi1aRJMmTWwdiohItaXFDURERGq4zz77zNYhiIjUGhrqJiIi\nYiPTpk1j+vTpADz55JN07doVgDVr1hAVFUXbtm05evQoFosFFxcXRo4ciZubGz169OD06dMA7N27\nl549e+Ln50doaCi7du2yWXtERKozJT4iIiI2EhoaSnJyMgDp6ekUFBRw7tw5kpOTCQsLK1M2JyeH\nxx57jO3bt9OkSROWLVsGwMMPP0xCQgIZGRnEx8czevToKm+HiEhNoKFuIiIiNuLn50dGRga//PIL\n9evXx9fXl/T0dJKTk5k+fTqvvvqqtayzszPe3t7WehaLhYKCAjZs2EBkZKS13JkzZ6q8HSIiNYES\nHxERERtxcHDA2dmZxMREOnXqhKenJ2vXriU3NxcXF5cyZX+7d4y9vT2nT5+mtLSUJk2akJWVVdWh\ni4jUOBrqJiIiYkOhoaHEx8cTFhZGaGgoM2bMwMfHB8Mwrli3cePGODs7s2TJEgBM0yQ7O7uyQxYR\nqZGU+IiIiNhQaGgoeXl5BAcH06JFCxwdHQkNDb3q+gsXLmT27Nl4eXnh5ubGihUrKjFaEZGaS/v4\niIiIiIhIjXW1+/iox0dERERERGo9JT4iIlKjdOrUCQCLxYK7u7uNoxERkZpCiY+IiNQoGzZssHUI\nIiJSAynxERGRauuNN97A3d0dd3d33nzzTQAaNWpk46hERKQm0j4+IiJSLWVkZDBnzhy+/fZbTNMk\nKCiIzp072zosERGpoZT4iIhItZSSksK9995Lw4YNAejfvz/Jyck2jkpERGoqDXUTEREREZFaT4mP\niIhUS6GhoSxfvpxTp05RWFjIxx9/fE0be4qIiPyWhrqJiEi15OvrS3R0NIGBgQDExMTg4+Nj46hE\nRKSmMkzTtHUMl+Tv72+mp6fbOgwREREREanGDMPIME3T/0rlNNRNRERERERqPSU+IiIi1dzKlSuZ\nOnUqALGxscTHxwMQHh6ORkeIiFwdzfERERGp5vr27Uvfvn1tHYaISI2mHh8REREbslgsdOzYkejo\naNq3b09UVBRJSUmEhITQrl07UlNTSUxMZMyYMZe9R2lpKdHR0bzwwgtVGLmISM2ixEdERMTGcnNz\nefrpp9m1axe7du1i0aJFpKSkEB8fz5QpU/6wbnFxMVFRUbRr1464uLgqilhEpOZR4iMi8ju/nUMh\nUhWcnZ3x8PDAzs4ONzc3unXrhmEYeHh4YLFY/rDuI488gru7O88//3zVBCsiUkMp8RGphSwWC+7u\n7n+6vj74i1St+vXrW4/t7Oys53Z2dhQXF/9h3U6dOrF27VqKiooqNUYRkZpOiY+IlHGlD1k12bx5\n8/D09MTLy4shQ4awatUqgoKC8PHxoXv37vz0008X1Zk1axZ33XUXp0+fZu/evfTs2RM/Pz9CQ0PZ\ntWsXAEuWLMHd3R0vLy/CwsKqullSxz300EP06tWL+++/v1b//RURKS+t6iZSS10Y95+ZmYmbmxvz\n5s0jPj6eVatWcfr0aTp16sTMmTMxDIPw8HC8vb1JSUlh0KBBZe6zd+9eHnvsMY4cOcL111/PrFmz\naN26NZ6enuzZswcHBwd++eUXvLy8rOfV0fbt24mLi2PDhg00a9aMn3/+GcMw2LRpE4Zh8J///IfX\nXnuN119/3Vrnrbfe4quvvmL58uXUr1+fhx9+mBkzZtCuXTu+/fZbRo8ezZo1a5g8eTJffvklrVu3\nJj8/34atlLrqqaee4sSJEwwZMoSFCxdiZ6fvNUVEfs8wTdPWMVySv7+/qb0JRP4ci8WCs7MzKSkp\nhISEMGLECFxdXRkxYgQ33ngjAEOGDOH+++/n7rvvJjw8HFdXV9555x3g/FC3Ro0a8cwzz9CtW7cy\nH/afe+451qxZw/Dhw+nXrx/33HMP7733Hrt37y6TNFQ3CQkJ/Pjjj7zyyivWa1u3buXpp58mLy+P\ns2fP4uzszBdffEFsbCwfffQRbdq0Yfny5Tg4OFBQUEDz5s3p0KGDtf6ZM2fYuXMnjz76KHv37uX+\n+++nf//+3HTTTbZoooiISJ1kGEaGaZr+Vyqnr4REaqk2bdoQEhICwODBg0lJSWHt2rUEBQXh4eHB\nmjVr2L59u7X8wIEDL7pHQUEBGzZsIDIyEm9vbx555BHy8vIAiImJYc6cOQDMmTOH4cOHV0GrKtbY\nsWMZM2YMW7duZebMmWXmSFyYVH7w4EHg/HLBTZo0ISsry/qzc+dOAGbMmEFcXBzff/89fn5+HDt2\nzCbtERERkctT4iNSSxmGcdH56NGjWbp0KVu3bmXkyJFlPug3bNjwonv80Yf9kJAQLBYL69ato6Sk\npFyLKVSFrl27smTJEmtS8vPPP3PixAlat24NwNy5c8uU9/HxYebMmfTt25dDhw7RuHFjnJ2dWbJk\nCQCmaZKdnQ2cHw4YFBTE5MmTad68Od9//30VtkxERESuhhIfkVrqwIEDbNy4EYBFixZxxx13ANCs\nWTMKCgpYunTpFe/xRx/2AYYOHcqDDz5YI3p73NzceP755+ncuTNeXl489dRTxMbGEhkZiZ+fH82a\nNbuozh133EF8fDy9e/fm6NGjLFy4kNmzZ+Pl5YWbmxsrVqwAYNy4cXh4eODu7k6nTp3w8vKq6uaJ\niIjIFWiOj0gtZLFY6NmzJ/7+/mRkZODq6sr8+fOZMmUKH3zwAbfccgvt27fnr3/9K7GxsYSHhxMf\nH4+///nhsb+d47Nv3z5GjRpFXl4e586d44EHHuDFF18E4Mcff8TZ2Zm8vDyaNGliyyaLiIhIHXW1\nc3yU+IjIn7Z06VJWrFjB/PnzbR2KiIiI1FFXm/hoOWsR+VPGjh3L559/zmeffWbrUERERESuSImP\niPwpCQkJtg5BRERE5KppcQMREakTLBZLtV99UEREKo8SHxERERERqfWU+IiISJ1RUlLCyJEjcXNz\no0ePHpw+fZqsrCxuv/12PD09uffeezl+/DgA4eHhjB8/nsDAQNq3b09ycrL1HuPGjSMgIABPT09m\nzpwJnF/effny5dZnRUVFWZc8FxER21PiIyIidUZOTg6PPfYY27dvp0mTJixbtoyhQ4fyr3/9iy1b\ntuDh4cFLL71kLV9cXExqaipvvvmm9frs2bNxcnIiLS2NtLQ0Zs2axb59+3jooYdITEwE4MSJE2zY\nsIHevXvbopkiInIJSnxERKTOcHZ2xtvbGwA/Pz/27t1Lfn4+nTt3BmDYsGGsX7/eWr5///7WshaL\nBYDVq1czb948vL29CQoK4tixY+Tk5NC5c2dycnI4cuQIH3zwAffddx/16mkNIRGR6kL/RxYRkTqj\nfv361mN7e3vy8/Ovqry9vT3FxcUAmKZJQkICERERF5UfOnQoCxYs4L///S9z5sypwMhFRKS81OMj\nIiJ1lpOTE02bNrXO35k/f7619+dyIiIiePfddzl37hwAe/bsobCwEIDo6GjefPNNAFxdXSsxchER\nuVbq8RERkTpt7ty5PProo5w6dYrbbrvtij01MTExWCwWfH19MU2T5s2bWxc1aNGiBS4uLtxzzz1V\nEbqIiFwDwzRNW8dwSf7+/mZ6erqtwxAREblqp06dwsPDg8zMTJycnGwdjohInWAYRoZpmv5XKqeh\nbiIiIhUgKSkJFxcXxo4dq6RHRKQa0lA3ERGRCtC9e3f2799v6zBEROQy1OMjIiIiIiK1nhIfERER\nERGp9ZT4iIiIiIhIrafER0REREREaj0lPiIiIiIiUusp8RERERERkVqvXImPYRg3GobxlWEYOb/+\n2fQPyjY2DOOgYRhvleeZIiIiIiIi16q8PT4TgK9N02wHfP3r+eW8DKwv5/NERERERESuWXkTn37A\n3F+P5wL3XKqQYRh+QAtgdTmfJyIiIiIics3Km/i0ME0z79fjHzmf3JRhGIYd8DrwTDmfJSIiIiIi\n8qfUu1IBwzCSgFsu8dLzvz0xTdM0DMO8RLnRwGemaR40DONKz3oYeBjg1ltvvVJoIiIiIiIiV+WK\niY9pmt0v95phGD8ZhtHSNM08wzBaAocvUSwYCDUMYzTQCLjOMIwC0zQvmg9kmuZ7wHsA/v7+l0qi\nRERERERErtkVE58rWAkMA6b++ueK3xcwTTPqwrFhGNGA/6WSHhERERERkcpS3jk+U4E7DcPIAbr/\neo5hGP6GYfynvMGJiIiIiIhUBMM0q+eIMn9/fzM9Pd3WYYiIiIiISDVmGEaGaZr+VypX3h4fERER\nERGRak+Jj4hIHWWxWHB3d7/q8m+++SanTp2ynk+ZMqXM640aNaqw2ERERCqaEh8REbkqV0p8RERE\nqjMlPiIidVhxcTFRUVG4uLgwYMAATp06xddff42Pjw8eHh6MGDGCM2fOMH36dA4dOkSXLl3o0qUL\nEyZM4PTp03h7exMVFXXRfadNm0ZAQACenp5MmjQJgMLCQnr37o2Xlxfu7u4sXry4qpsrIiJ1WHmX\nsxYRkRps9+7dzJ49m5CQEEaMGMEbb7zBzJkz+frrr2nfvj1Dhw7l3Xff5YknnuCNN95g7dq1NGvW\nDIC33nqLrKysi+65evVqcnJySE1NxTRN+vbty/r16zly5AitWrXi008/BeDEiRNV2lYREanb1OMj\nIlKHtWnThpCQEAAGDx7M119/jbOzM+3btwdg2LBhrF+//pruuXr1alavXo2Pjw++vr7s2rWLnJwc\nPDw8+Oqrrxg/fjzJyck4OTlVeHtEREQuR4mPiNQZvXr1Ij8/39ZhVCuGYZQ5b9KkSbnvaZomzz33\nHFlZWWRlZZGbm8tDDz1E+/btyczMxMPDgxdeeIHJkyeX+1kiIiJXS4mPiNQJpmnyySefVMgH+9rk\nwIEDbNy4EYBFixbh7+/P/2/v7mOrOugwjj8PLzUBmS+DbMRNyiIrb4UqDSkRrUSgNSFoIkxFAksW\nCSwmyCJhZvyD/uM0voTIHxotmSRGtImyZBpUnFmwQlbQSDvjqPMaeVFgwaXNolb8+cc9Nl0p3FNo\n7zk99/tJGu659/Tkd5Mnl/uct5ZKJfX19UmSjhw5otbWVknSrFmz1N/fP/S706dP1+Dg4E3bbGtr\nU0dHhwYGBiRJFy9e1JUrV3Tp0iXNmDFDW7du1d69e3X27NmJfnsAAAyh+AAorFKppIaGBm3btk1L\nly7V1KlTde3aNZVKJS1cuPCmi/ol6cyZM2ptbdWKFSvU1tamy5cvZ/wuJlZDQ4MOHTqkRYsW6fr1\n69qzZ48OHz6szZs3q7GxUVOmTNHOnTslSTt27FB7e7vWrFkztLxs2bKbbm6wfv16bdmyRatWrVJj\nY6M2bdqk/v5+nTt3TitXrlRTU5MOHDig/fv3V/39AgBqlyMi6xlG1dzcHN3d3VmPAWASK5VKeuih\nh9TV1aWWlhbV19eru7tbAwMDmj9/vk6ePDl0Uf/ixYu1e/dutba26tixY5ozZ46OHj2q48ePq6Oj\nI+u3AgAAbsH2mYhorrQed3UDUGjz5s1TS0vLTc+PvKj/4MGDam9vV09Pj9atWydJunHjhubOnVvV\neQEAwMSg+AAotJkzZ476/MiL+m0rIrRkyZKha14AAEBxcI0PgJo08qL+1atXq6GhQVevXh16fnBw\nUL29vVmOCQAAxgnFB0BNGnlR/65du1RXV6fOzk7t27dPy5cvV1NTk7q6urIeFQAAjANubgCg5pRK\nJW3YsEE9PT1ZjwIAAO5S2psbcMQHAAAAQOFRfADUnPr6eo72AABQYyg+AAAAAAqP4gMAAACg8Cg+\nAAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg\n8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMA\nAACg8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg8Cg+AAAAAAqP4gMAAACg8BwRWc8wKttX\nJf0l6zlyZraka1kPgUmFzGCsyAzGiszgTpAbjNXtMjMvIuZU2kBuiw9uZrs7IpqzngOTB5nBWJEZ\njBWZwZ0gNxir8cgMp7oBAAAAKDyKDwAAAIDCo/hMLt/KegBMOmQGY0VmMFZkBneC3GCs7jozXOMD\nAAAAoPA44gMAAACg8Cg+OWb77bZ/bvt88u/bbrPuPbYv2P5GNWdEvqTJjO0m27+x3Wv797Y/lsWs\nyJbtdtt/tN1n+8lRXn+T7aPJ66dt11d/SuRJisw8Yful5HPlhO15WcyJ/KiUmWHrfdR22OYub0iV\nG9uPJJ83vba/l3bbFJ98e1LSiYhYIOlEsnwrX5D0QlWmQp6lyczrkrZFxBJJ7ZK+bvutVZwRGbM9\nVdIhSR+StFjSJ2wvHrHaY5KuR8S7JH1N0tPVnRJ5kjIzv5XUHBHLJHVK+lJ1p0SepMyMbM+StFvS\n6epOiDxKkxvbCyR9TtJ7k+8yn0m7fYpPvn1Y0jPJ42ckfWS0lWyvkHSfpJ9VaS7kV8XMRMTLEXE+\neXxJ0hVJFf/oFwplpaS+iHglIv4t6fsqZ2e44VnqlPRB267ijMiXipmJiOcj4vVk8ZSkB6o8I/Il\nzeeMVN5x+7Skf1ZzOORWmtx8StKhiLguSRFxJe3GKT75dl9EXE4e/03lcvMGtqdI+oqkz1ZzMORW\nxcwMZ3ulpDpJf5rowZAr75D012HLF5LnRl0nIv4j6TVJ91ZlOuRRmswM95ikn07oRMi7ipmx/R5J\nD0bEc9UcDLmW5rPmYUkP2/617VO229NufNo4DIi7YPsXku4f5aWnhi9ERNge7RZ8j0v6SURcYGds\nbRiHzPx/O3MlHZG0PSL+O75TAqhVtrdKapbUmvUsyK9kx+1XJT2a8SiYfKZJWiDpAyofWX7BdmNE\n/CPNLyJDEbH2Vq/Z/rvtuRFxOfmSOtqhvFWS3mf7cUlvllRneyAibnc9ECaxcciMbN8j6TlJT0XE\nqQkaFfl1UdKDw5YfSJ4bbZ0LtqdJeoukV6szHnIoTWZke63KO2FaI+JfVZoN+VQpM7MkLZX0q2TH\n7f2SnrW9MSK6qzYl8ibNZ80FSacjYlDSn22/rHIRerHSxjnVLd+elbQ9ebxd0rGRK0TEJyPinRFR\nr/Lpbt+l9NS0ipmxXSfpRypnpbOKsyE/XpS0wPb8JA8fVzk7ww3P0iZJvwz+8Fstq5gZ2++W9E1J\nG8dyzj0K67aZiYjXImJ2RNQn32FOqZwdSk9tS/P/049VPtoj27NVPvXtlTQbp/jk2xclrbN9XtLa\nZFm2m21/O9PJkFdpMvOIpPdLetT275KfpmzGRRaSa3Y+Lem4pD9I+kFE9Nr+vO2NyWrfkXSv7T5J\nT+j2d5VEwaXMzJdVPvPgh8nnysgvK6ghKTMDvEHK3ByX9KrtlyQ9L2lvRKQ6I8HswAMAAABQdBzx\nAQAAAFB4FB8AAAAAhUfxAQAAAFB4FB8AAAAAhUfxAQAAAFB4FB8AAAAAhUfxAQAAAFB4FB8AAAAA\nhfc/T72Ed4Z7X7UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1350fa438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "_, ax = plt.subplots(figsize=(14,10))\n",
    "ax.scatter(embeddings[:,0], embeddings[:,1], alpha=0)\n",
    "for i in range(len(vectors)):\n",
    "    ax.annotate(f_tokens[i], ((embeddings[i,0], embeddings[i,1])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**What kind of clusters of food-themed terms can you notice?**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Doc2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "---\n",
    "Doc2Vec, the most powerful extension of word2vec\n",
    "---\n",
    "\n",
    "Doc2vec (aka paragraph2vec or sentence embeddings) extrapolates the word2vec algorithm to larger blocks of text, such as sentences, paragraphs or entire documents. \n",
    "\n",
    "![](images/overview_word.png)\n",
    "\n",
    "![](images/overview_paragraph.png)\n",
    "\n",
    "Every paragraph is mapped to a unique vector, represented by a column in matrix D and every word is also mapped to a unique vector, represented by a column in matrix W . \n",
    "The paragraph vector and word vectors are averaged or concatenated to predict the next word in a context. \n",
    "\n",
    "Each additional context does not have be a fixed length (because it is vectorized and projected into the same space).\n",
    "\n",
    "Additional parameters but the updates are sparse thus still efficent.\n",
    "\n",
    "__2 architectures__:\n",
    "\n",
    "1. Distrubted Memory (DM)\n",
    "2. Distrubted Bag of Words (DBOW)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distrubted Memory (DM)\n",
    "\n",
    "__Highlights__:\n",
    "\n",
    "- Assign and randomly initialize paragraph vector for each doc\n",
    "- Predict next word using context words and paragraph vector\n",
    "- Slide context window across doc but keep paragraph vector fixed (hence: Distrubted Memory)\n",
    "- Update weights via SGD and backprop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distrubted Bag of Words (DBOW)\n",
    "\n",
    "__Highlights__:\n",
    "\n",
    "- ONLY use paragraph vectors (no word vectors)\n",
    "- Take a window of words in a paragraph and randomly sample which ones to predict using paragraph vector\n",
    "- Simpler, more memory effecient\n",
    "\n",
    "![](images/DBOW.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try building our own Doc2Vec model with Gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Doc2Vec Parameters\n",
    "- `size`: Number of dimensions for the embedding model\n",
    "- `window`: Number of context words to observe in each direction within a document\n",
    "- `min_count`: Minimum frequency for words included in model\n",
    "- `dm` (distributed memory): '0' indicates DBOW model; '1' indicates DM\n",
    "- `alpha`: Learning rate (initial); prevents model from over-correcting, enables finer tuning\n",
    "- `iter`: Number of iterations through corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "collecting all words and their counts\n",
      "PROGRESS: at example #0, processed 0 words (0/s), 0 word types, 0 tags\n",
      "PROGRESS: at example #10000, processed 1431895 words (4396683/s), 60881 word types, 10000 tags\n",
      "collected 63348 word types and 10788 unique tags from a corpus of 10788 examples and 1548468 words\n",
      "Loading a fresh vocabulary\n",
      "min_count=5 retains 15255 unique words (24% of original 63348, drops 48093)\n",
      "min_count=5 leaves 1472770 word corpus (95% of original 1548468, drops 75698)\n",
      "deleting the raw counts dictionary of 63348 items\n",
      "sample=0.001 downsamples 45 most-common words\n",
      "downsampling leaves estimated 1118059 word corpus (75.9% of prior 1472770)\n",
      "estimated required memory for 15255 words and 300 dimensions: 57185100 bytes\n",
      "resetting layer weights\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Doc2Vec\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "from nltk.corpus import reuters\n",
    "\n",
    "# Tokenize Reuters corpus\n",
    "tokenized_docs = [nltk.word_tokenize(reuters.raw(fileid)) for fileid in reuters.fileids()]\n",
    "\n",
    "# Convert tokenized documents to TaggedDocuments\n",
    "tagged_docs = [TaggedDocument(doc, tags=[idx]) for idx, doc in enumerate(tokenized_docs)]\n",
    "\n",
    "# Create and train the doc2vec model. May take a few seconds\n",
    "doc2vec = Doc2Vec(size=300, window=5, min_count=5, dm = 1, iter=10)\n",
    "\n",
    "# Build the word2vec model from the corpus\n",
    "doc2vec.build_vocab(tagged_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also fortify your Doc2Vec models with pre-trained Word2Vec models.\n",
    "Let's try re-training with GoogleNews-trained word vectors.  \n",
    "Download [here](https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?usp=sharing)  \n",
    "(Size is 1.5gb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This may take a few minutes to run\n",
    "w2v_loc = # your saved location of GoogleNews-vectors-negative300.bin.gz\n",
    "doc2vec.intersect_word2vec_format(w2v_loc, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training model with 3 workers on 15255 vocabulary and 300 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
      "PROGRESS: at 4.56% examples, 501506 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 9.25% examples, 518806 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 14.56% examples, 541083 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 19.44% examples, 545440 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 24.75% examples, 553349 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 28.82% examples, 537311 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 32.49% examples, 519493 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 36.55% examples, 511619 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 40.50% examples, 504437 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 45.47% examples, 507292 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 50.50% examples, 513053 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 55.35% examples, 514623 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 59.87% examples, 515344 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 64.39% examples, 512104 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 68.21% examples, 506879 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 73.00% examples, 507863 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 78.15% examples, 512007 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 83.46% examples, 516562 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 88.90% examples, 521565 words/s, in_qsize 6, out_qsize 0\n",
      "PROGRESS: at 94.39% examples, 525719 words/s, in_qsize 5, out_qsize 0\n",
      "PROGRESS: at 99.62% examples, 528600 words/s, in_qsize 5, out_qsize 0\n",
      "worker thread finished; awaiting finish of 2 more threads\n",
      "worker thread finished; awaiting finish of 1 more threads\n",
      "worker thread finished; awaiting finish of 0 more threads\n",
      "training on 15484680 raw words (11287240 effective words) took 21.4s, 528654 effective words/s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "11287240"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc2vec.train(tagged_docs, epochs=10, total_examples=doc2vec.corpus_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using what you have learned from Week 1 and today's lecture, build a binary classifier for the movie reviews dataset in `data/movie_reviews.csv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "248px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": true,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}