[NLP] Generative model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/masdevas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import scrapy\n",
    "import scrapy.crawler as crawler\n",
    "from scrapy.utils.log import configure_logging\n",
    "from multiprocessing import Process, Queue\n",
    "from twisted.internet import reactor\n",
    "from collections import Counter\n",
    "import logging\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import copy\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from datasets import load_dataset\n",
    "from datasets.arrow_dataset import Dataset\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "from transformers.models.gpt2.modeling_gpt2 import GPT2LMHeadModel\n",
    "from transformers import TrainingArguments, Trainer\n",
    "from transformers import pipeline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import CrossEntropyLoss\n",
    "from torch.optim import AdamW\n",
    "from torch.optim.lr_scheduler import ExponentialLR\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import BatchSampler, RandomSampler\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "from lyricsgenius import Genius\n",
    "import pickle\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import wordpunct_tokenize\n",
    "nltk.download('punkt')\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from  matplotlib.ticker import FuncFormatter\n",
    "\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "logger = logging.getLogger('nlp')\n",
    "logger.setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fetch sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_web_data(function):\n",
    "    def f(q):\n",
    "        try:\n",
    "            json_data = function()\n",
    "            q.put(json.dumps(json_data))\n",
    "        except Exception as e:\n",
    "            q.put(e)\n",
    "    q = Queue()\n",
    "    p = Process(target=f, args=(q,))\n",
    "    p.start()\n",
    "    result = q.get()\n",
    "    p.join()\n",
    "    gc.collect()\n",
    "    try:\n",
    "        json_data = json.loads(result)\n",
    "    except:\n",
    "        raise result\n",
    "    return json_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rap texts parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_genius_token():\n",
    "    with open('genius_token.txt', 'r') as f:\n",
    "        token = f.readline().strip()\n",
    "    return token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rap_data_extended():\n",
    "    genius = Genius(read_genius_token(), timeout=60)\n",
    "    storage = {}\n",
    "    try:\n",
    "        class RapExtendedSpider(scrapy.Spider):\n",
    "            name = \"rap_extended_checker\"\n",
    "            start_urls = [\n",
    "                'https://www.allmusic.com/style/gangsta-rap-ma0000002611/artists?1651103019828'\n",
    "            ]\n",
    "\n",
    "            def parse(self, response, depth=0):\n",
    "                resp = response.xpath(\"//div[@class='artist-highlight info-grid']//div[@class='artist']/text()\")\n",
    "                for item in resp:\n",
    "                    artist_name = item.get().strip()\n",
    "                    if len(artist_name) > 0:\n",
    "                        try:\n",
    "                            artist = genius.search_artist(artist_name, max_songs=150)\n",
    "                        except:\n",
    "                            continue\n",
    "                        storage[artist_name] = []\n",
    "                        for song in artist.songs:\n",
    "                            storage[artist_name].append(song.lyrics)\n",
    "                \n",
    "        runner = crawler.CrawlerRunner()\n",
    "        deferred = runner.crawl(RapExtendedSpider)\n",
    "        deferred.addBoth(lambda _: reactor.stop())\n",
    "        reactor.run()\n",
    "        return storage\n",
    "    except Exception as e:\n",
    "        return {'exception' : str(e)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOADING\n"
     ]
    }
   ],
   "source": [
    "storage_file = 'storage.pkl'\n",
    "def is_rawdata_exist(storage_file):\n",
    "    return os.path.exists(storage_file)\n",
    "    # return False\n",
    "\n",
    "if is_rawdata_exist(storage_file):\n",
    "    print('LOADING')\n",
    "    with open(storage_file, 'rb') as f:\n",
    "        storage = pickle.load(f)\n",
    "else:\n",
    "    print('FROM WEB')\n",
    "    storage = {}  \n",
    "#     storage['usagov'] = get_web_data(get_usagov_rss_data)\n",
    "#     storage['joke'] = get_web_data(get_joke_rss_data)\n",
    "#     storage['rap'] = get_web_data(get_rap_data)\n",
    "    storage['rap'] = get_web_data(get_rap_data_extended)\n",
    "    with open(storage_file, 'wb') as f:\n",
    "        pickle.dump(storage, f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess sentences for HuggingFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_to_process = copy.deepcopy(storage)\n",
    "\n",
    "map_labels = {'rap' : 0}\n",
    "\n",
    "words_limit = 22\n",
    "words_at_least = 10\n",
    "\n",
    "stop_symbols = set(['...', '.', '?', '!', '!!!', '?!', '?!'])\n",
    "\n",
    "def join_rap_sentences(local_lines, lines_in_group):\n",
    "    rap_sentences = []\n",
    "    idx = 0\n",
    "    while idx < len(local_lines):\n",
    "        line = local_lines[idx].strip().lower()\n",
    "        if len(line) == 0:\n",
    "            del local_lines[idx]\n",
    "            continue\n",
    "        if idx + lines_in_group > len(local_lines):\n",
    "            res_proc_item = line\n",
    "        else:\n",
    "            second_line = local_lines[idx + 1].strip().lower()\n",
    "            if len(second_line) == 0:\n",
    "                del local_lines[idx + 1]\n",
    "                continue\n",
    "            res_proc_item = line + ' ' + second_line\n",
    "        res_proc_item = res_proc_item.replace(',', '')\n",
    "        splitted = res_proc_item.split(' ')[:words_limit]\n",
    "        if len(splitted) >= words_at_least:\n",
    "            joined = ' '.join(splitted)\n",
    "            end_empty = True\n",
    "            for token in stop_symbols:\n",
    "                if joined.endswith(token):\n",
    "                    end_empty = False\n",
    "                    break\n",
    "            if end_empty:\n",
    "                joined += '.'\n",
    "            \n",
    "            rap_sentences.append(joined)\n",
    "        idx += lines_in_group\n",
    "    return rap_sentences\n",
    "\n",
    "# TODO how to have a deal with slang?\n",
    "def get_approx_sentences_from_lyrics(lyrics, local_processed_data):\n",
    "    couplets = re.split('\\[.*\\]' ,lyrics)\n",
    "    for couplet in couplets:\n",
    "        local_lines = couplet.split('\\n')\n",
    "        \n",
    "        if len(local_lines) == 1:\n",
    "            continue\n",
    "        lines_in_group = 2\n",
    "        rap_sentences = join_rap_sentences(local_lines, lines_in_group)\n",
    "        local_processed_data.extend(rap_sentences)\n",
    "\n",
    "def get_data_lyrics(substorage, topic_key):\n",
    "    local_processed_data = []\n",
    "    for lyrics_list in substorage.values():\n",
    "        for lyrics in lyrics_list:\n",
    "            get_approx_sentences_from_lyrics(lyrics, local_processed_data)\n",
    "    local_processed_labels = [map_labels[topic_key]] * len(local_processed_data)\n",
    "    with open(f'tmp_{topic_key}.json', 'w') as f:\n",
    "        json.dump({'section' : local_processed_data}, f, indent=4, sort_keys=True)\n",
    "    return local_processed_data, local_processed_labels\n",
    "\n",
    "processed_data = []\n",
    "processed_labels = []\n",
    "for topic_key in map_labels.keys():\n",
    "    if topic_key == 'rap':\n",
    "        local_processed_data, local_processed_labels = get_data_lyrics(storage_to_process[topic_key], topic_key)\n",
    "        processed_data.extend(local_processed_data)\n",
    "        processed_labels.extend(local_processed_labels)\n",
    "    else:\n",
    "        raise Exception(f'Unknown topic_key: {topic_key}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_text_data = pd.DataFrame({'text' : processed_data, 'label' : processed_labels})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i am a nightmare walkin' psychopath talkin' ki...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>livin' life like a firecracker quick is my fus...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>red or blue cuz or blood it just don't matter ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>the gangs of l.a. will never die - just multiply.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>you see they hit us then we hit them then we h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138314</th>\n",
       "      <td>with they **** ***** attitudes why it's gotta ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138315</th>\n",
       "      <td>i don't owe you chick you wasn't with me when ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138316</th>\n",
       "      <td>even if you was you wouldn't shoot the stick w...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138317</th>\n",
       "      <td>know you a hatin lil **** always talking other...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138318</th>\n",
       "      <td>need to put a sock in it you speak facts alrig...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>138319 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  label\n",
       "0       i am a nightmare walkin' psychopath talkin' ki...      0\n",
       "1       livin' life like a firecracker quick is my fus...      0\n",
       "2       red or blue cuz or blood it just don't matter ...      0\n",
       "3       the gangs of l.a. will never die - just multiply.      0\n",
       "4       you see they hit us then we hit them then we h...      0\n",
       "...                                                   ...    ...\n",
       "138314  with they **** ***** attitudes why it's gotta ...      0\n",
       "138315  i don't owe you chick you wasn't with me when ...      0\n",
       "138316  even if you was you wouldn't shoot the stick w...      0\n",
       "138317  know you a hatin lil **** always talking other...      0\n",
       "138318  need to put a sock in it you speak facts alrig...      0\n",
       "\n",
       "[138319 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_text_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22    19892\n",
       "18    12911\n",
       "17    12609\n",
       "16    12526\n",
       "19    11595\n",
       "15    11168\n",
       "20    10220\n",
       "14     9938\n",
       "13     8425\n",
       "21     8292\n",
       "12     7880\n",
       "11     6694\n",
       "10     6169\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_text_data['text'].apply(lambda x: len(x.split(' '))).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyse common starts and continuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_subseqs(data, len_main_subseq, len_extra_subseq, count_main_seqs, count_extra_seqs):\n",
    "    set_main_seqs = {}\n",
    "    for line in data['text']:\n",
    "        splited_sentence = line[:-1].split(' ')\n",
    "        for word_idx in range(len(splited_sentence)):\n",
    "            if word_idx + len_main_subseq + len_extra_subseq > len(splited_sentence):\n",
    "                break\n",
    "            main_subseq = ' '.join(splited_sentence[word_idx:word_idx+len_main_subseq])\n",
    "            if main_subseq not in set_main_seqs.keys():\n",
    "                set_main_seqs[main_subseq] = 1\n",
    "            else:\n",
    "                set_main_seqs[main_subseq] += 1\n",
    "    sorted_main_seqs_counted = sorted(list(set_main_seqs.items()), key=lambda x: -x[1])\n",
    "    sorted_main_seqs = [sorted_main_seqs_counted[idx] for idx in range(count_main_seqs)]\n",
    "    \n",
    "    preresult = {key : [] for key in sorted_main_seqs}\n",
    "    for main_key, count in sorted_main_seqs:\n",
    "        splitted_main_key = main_key.split(' ')\n",
    "        for line in data['text']:\n",
    "            splited_sentence = line[:-1].split(' ')\n",
    "            for word_idx in range(len(splited_sentence)):\n",
    "                if word_idx + len_main_subseq + len_extra_subseq > len(splited_sentence):\n",
    "                    break\n",
    "                is_main_eq = True\n",
    "                for main_word, extra_word in zip(splitted_main_key, splited_sentence[word_idx:word_idx+len_main_subseq]):\n",
    "                    if main_word != extra_word:\n",
    "                        is_main_eq = False\n",
    "                        break\n",
    "                if not is_main_eq:\n",
    "                    continue\n",
    "                extra_subseq = ' '.join(splited_sentence[word_idx+len_main_subseq:word_idx+len_main_subseq+len_extra_subseq])\n",
    "                preresult[(main_key, count)].append(extra_subseq)\n",
    "    \n",
    "    result = {key : [] for key in sorted_main_seqs}\n",
    "    for key, collection in preresult.items():\n",
    "        common_extras = Counter(collection).most_common(count_extra_seqs)\n",
    "        result[key] = common_extras\n",
    "    return result\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('in the', 11599): [('hood', 424),\n",
       "  ('back', 404),\n",
       "  ('house', 399),\n",
       "  ('air', 298),\n",
       "  ('game', 296)],\n",
       " ('on the', 6422): [('block', 280),\n",
       "  ('streets', 192),\n",
       "  ('floor', 156),\n",
       "  ('mic', 136),\n",
       "  ('corner', 125)],\n",
       " ('to the', 5550): [('top', 101),\n",
       "  ('beat', 96),\n",
       "  ('game', 74),\n",
       "  ('west', 72),\n",
       "  ('side', 67)],\n",
       " ('i got', 4540): [('a', 753),\n",
       "  ('the', 477),\n",
       "  ('my', 381),\n",
       "  ('to', 249),\n",
       "  ('it', 169)],\n",
       " ('you know', 4513): [('what', 693),\n",
       "  ('i', 464),\n",
       "  ('the', 332),\n",
       "  ('how', 258),\n",
       "  ('that', 227)]}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyse_subseqs(df_text_data, 2, 1, 5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>back and forth with the gangster glide started...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>when i'm all alone 'cause you're not home (woo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>got cris' on pop henny wit no chaser and mami ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>and we got the workers choppin' rock fiends by...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>on on on rollin on you better keep rollin on.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11885</th>\n",
       "      <td>i'm heavy in the hood 'cause i got that weight...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11886</th>\n",
       "      <td>i'm heavy in the hood 'cause i got that weight...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11887</th>\n",
       "      <td>(what up elder?) i'm heavy in the hood 'cause ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11888</th>\n",
       "      <td>(they already know they can't keep up with you...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11889</th>\n",
       "      <td>see me pushin' through don't hate don't hate (...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11890 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "0      back and forth with the gangster glide started...      0\n",
       "1      when i'm all alone 'cause you're not home (woo...      0\n",
       "2      got cris' on pop henny wit no chaser and mami ...      0\n",
       "3      and we got the workers choppin' rock fiends by...      0\n",
       "4          on on on rollin on you better keep rollin on.      0\n",
       "...                                                  ...    ...\n",
       "11885  i'm heavy in the hood 'cause i got that weight...      0\n",
       "11886  i'm heavy in the hood 'cause i got that weight...      0\n",
       "11887  (what up elder?) i'm heavy in the hood 'cause ...      0\n",
       "11888  (they already know they can't keep up with you...      0\n",
       "11889  see me pushin' through don't hate don't hate (...      0\n",
       "\n",
       "[11890 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_data = df_text_data[df_text_data['text'].str.contains('in the hood') | df_text_data['text'].str.contains('in the back') | df_text_data['text'].str.contains('in the house') | df_text_data['text'].str.contains('in the air') | df_text_data['text'].str.contains('in the game')]\n",
    "sampled_index = chosen_data.index\n",
    "subsamples_data = df_text_data.drop(index=sampled_index).sample(n=10000)\n",
    "subsamples_data = pd.concat([subsamples_data, chosen_data], ignore_index=True)\n",
    "subsamples_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('in the', 2831): [('hood', 424),\n",
       "  ('back', 404),\n",
       "  ('house', 399),\n",
       "  ('air', 298),\n",
       "  ('game', 296)],\n",
       " ('on the', 583): [('streets', 34),\n",
       "  ('block', 28),\n",
       "  ('street', 14),\n",
       "  ('mic', 14),\n",
       "  ('floor', 13)],\n",
       " ('to the', 451): [('top', 12),\n",
       "  ('beat', 8),\n",
       "  ('streets', 8),\n",
       "  ('head', 7),\n",
       "  ('side', 7)],\n",
       " ('i got', 413): [('a', 56),\n",
       "  ('the', 45),\n",
       "  ('my', 44),\n",
       "  ('that', 30),\n",
       "  ('to', 22)],\n",
       " ('the hood', 410): [(\"'cause\", 21),\n",
       "  ('i', 20),\n",
       "  ('they', 19),\n",
       "  ('and', 15),\n",
       "  ('are', 12)]}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyse_subseqs(subsamples_data, 2, 1, 5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "generation_start = \"in the\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HuggingFace transformer is coming!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_dataset = Dataset.from_pandas(chosen_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label', '__index_level_0__'],\n",
       "    num_rows: 1890\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': \"so you're in the back of me just listen and learn because you'll all get a chance or maybe a turn.\",\n",
       " 'label': 0,\n",
       " '__index_level_0__': 76418}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_dataset[1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = 'gpt2'\n",
    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
    "type(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85f1d9d411e544238523e2a079fa8fe8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1890 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding='max_length', truncation=True, max_length=words_limit, return_tensors='pt')\n",
    "\n",
    "tokenized_datasets = hf_dataset.map(tokenize_function)\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Item #0: in the middle of the night, and I was just sitting there, and I was like, 'Oh my\n",
      " Item #1: in the middle of the night.\"I'm not sure if I'm going to be able to\n",
      " Item #2: in the middle of the night.\"I'm not sure if I'm going to be able to\n",
      " Item #3: in the middle of the night.\"I'm not going to be able to sleep tonight,\" he\n",
      " Item #4: in the middle of the night, and I was in the middle of the night. I was in the middle\n",
      " Item #5: in the middle of the night, and I was in the middle of the night. I was in the middle\n",
      " Item #6: in the middle of the night, and I was in the middle of the night, and I was in the\n",
      " Item #7: in the middle of the night, and I was in a state of shock. I was in a state of\n",
      " Item #8: in the middle of the night, and I was just sitting there, and I was like, 'Oh my\n",
      " Item #9: in the middle of the night.\"I'm not sure if I'm going to be able to\n"
     ]
    }
   ],
   "source": [
    "cpu = torch.device(\"cpu\")\n",
    "device = torch.device(\"cuda\")\n",
    "\n",
    "def generate_data(generation_start, model, tokenizer, num_return_sequences):\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()    \n",
    "    model_cpu = model.to(cpu)\n",
    "    generator = pipeline('text-generation', model=model_cpu, tokenizer=tokenizer, temperature=0.1)\n",
    "    generated = generator(generation_start, max_length=words_limit, num_return_sequences=num_return_sequences)\n",
    "    for idx, item in enumerate(generated):\n",
    "        print(f' Item #{idx}:', item['generated_text'].strip().replace('\\n',''))\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "    \n",
    "\n",
    "generate_data(generation_start, model, tokenizer, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_pretrained(\"model_storage\")\n",
    "model.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9e311f08ce24c548acc63109e371366",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 started. Eval on train: 20455.9453125. Cum norm: 9528.4931640625\n",
      "Epoch 1 started. Eval on train: 20455.61328125. Cum norm: 9528.5283203125\n",
      "Epoch 2 started. Eval on train: 20452.052734375. Cum norm: 9528.5703125\n",
      "Epoch 3 started. Eval on train: 20350.099609375. Cum norm: 9528.6025390625\n",
      "Epoch 4 started. Eval on train: 20179.2109375. Cum norm: 9528.6328125\n",
      "Epoch 5 started. Eval on train: 20291.38671875. Cum norm: 9528.671875\n",
      "Epoch 6 started. Eval on train: 20376.5546875. Cum norm: 9528.703125\n",
      "Epoch 7 started. Eval on train: 20407.05078125. Cum norm: 9528.73046875\n",
      "Epoch 8 started. Eval on train: 20407.998046875. Cum norm: 9528.7529296875\n",
      "Epoch 9 started. Eval on train: 20406.02734375. Cum norm: 9528.7734375\n",
      "Epoch 10 started. Eval on train: 20404.48828125. Cum norm: 9528.79296875\n",
      "Epoch 11 started. Eval on train: 20409.623046875. Cum norm: 9528.80859375\n",
      "Epoch 12 started. Eval on train: 20405.83203125. Cum norm: 9528.82421875\n",
      "Epoch 13 started. Eval on train: 20394.93359375. Cum norm: 9528.83203125\n",
      "Epoch 14 started. Eval on train: 20392.654296875. Cum norm: 9528.84375\n",
      "Epoch 15 started. Eval on train: 20393.94921875. Cum norm: 9528.853515625\n",
      "Epoch 16 started. Eval on train: 20380.94921875. Cum norm: 9528.859375\n",
      "Epoch 17 started. Eval on train: 20387.744140625. Cum norm: 9528.8662109375\n",
      "Epoch 18 started. Eval on train: 20383.115234375. Cum norm: 9528.8720703125\n",
      "Epoch 19 started. Eval on train: 20381.12890625. Cum norm: 9528.876953125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Item #0: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #1: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #2: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #3: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #4: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #5: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #6: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #7: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #8: in the the the the the the the the the the the the the the the the the the the the the\n",
      " Item #9: in the the the the the the the the the the the the the the the the the the the the the\n"
     ]
    }
   ],
   "source": [
    "trained_model = GPT2LMHeadModel.from_pretrained(\"model_storage\").to(device)\n",
    "\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()\n",
    "def train():\n",
    "    def check_weights_cum_norm(model):\n",
    "        cum_norm = 0\n",
    "        for name, W in model.named_parameters():\n",
    "            cum_norm += W.norm(2)\n",
    "        return cum_norm\n",
    "\n",
    "    def eval_on_loader(trained_model, loader):\n",
    "        with torch.no_grad():\n",
    "            losses = []\n",
    "            for batch in train_dataloader:\n",
    "                ids = torch.stack(batch['input_ids'][0]).to(device)\n",
    "                outputs = trained_model(input_ids=ids)\n",
    "                logits = outputs['logits']\n",
    "                sentences = logits.transpose(0, 1)\n",
    "                labels = ids.transpose(0, 1)\n",
    "\n",
    "                for sentence, label in zip(sentences, labels):\n",
    "                    sentence_cloned = sentence.clone()\n",
    "                    label_cloned = label.clone()\n",
    "                    torch.nn.functional.relu(sentence_cloned, inplace=True)\n",
    "                    losses.append(criterion(sentence_cloned, label_cloned))\n",
    "            return sum(losses)\n",
    "\n",
    "    batch_size = 32\n",
    "    train_dataloader = DataLoader(tokenized_datasets, shuffle=True, batch_size=batch_size)\n",
    "    optimizer = AdamW(trained_model.parameters(), lr=0.000001)\n",
    "    scheduler = ExponentialLR(optimizer, gamma=0.85)\n",
    "\n",
    "    criterion = CrossEntropyLoss()\n",
    "    num_epochs = 20\n",
    "    progress_bar_epoch = tqdm(range(num_epochs))\n",
    "\n",
    "    num_batches = len(tokenized_datasets) // batch_size\n",
    "    if len(tokenized_datasets) % batch_size:\n",
    "        num_batches += 1\n",
    "    gpt2_losses = []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_loss = eval_on_loader(trained_model, train_dataloader)\n",
    "        print(f'Epoch {epoch} started. Eval on train: {train_loss}. Cum norm: {check_weights_cum_norm(trained_model)}')\n",
    "        gpt2_losses.append(train_loss.item())\n",
    "        for batch in train_dataloader:\n",
    "            optimizer.zero_grad()\n",
    "            ids = torch.stack(batch['input_ids'][0]).to(device)\n",
    "            outputs = trained_model(input_ids=ids)\n",
    "            logits = outputs['logits']\n",
    "            sentences = logits.transpose(0, 1)\n",
    "            labels = ids.transpose(0, 1)\n",
    "            losses = []\n",
    "            for sentence, label in zip(sentences, labels):\n",
    "                sentence_cloned = sentence.clone()\n",
    "                label_cloned = label.clone()\n",
    "                with torch.no_grad():\n",
    "                    torch.nn.functional.relu(sentence_cloned, inplace=True)\n",
    "                losses.append(criterion(sentence_cloned, label_cloned))\n",
    "\n",
    "            res_loss = sum(losses) / len(losses)\n",
    "\n",
    "            res_loss.backward()\n",
    "            torch.nn.utils.clip_grad_norm_(trained_model.parameters(), 0.25)\n",
    "            optimizer.step()\n",
    "            del ids, outputs, logits, sentences, labels\n",
    "            gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "        progress_bar_epoch.update(1)\n",
    "        scheduler.step()\n",
    "    generate_data(generation_start, trained_model, tokenizer, 10)\n",
    "    return gpt2_losses\n",
    "\n",
    "gpt2_losses = train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Loss for epoch')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnEAAAGDCAYAAABEP0a3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABAvElEQVR4nO3deZyddXn//9c1+8yZyWxnEibrOQNhiSwhhAwKgoBatK1gtbgVcUWrtlr9tlDrr1q1/WIXarXVioBCCyoVrHxdQRKUNZCEEJYkJGQmG0lmTWbL7Nfvj3NPchImYZLMOfdZ3s/HYx5z5nPf5z7XOSTMO/fn/ly3uTsiIiIikl0Kwi5ARERERI6dQpyIiIhIFlKIExEREclCCnEiIiIiWUghTkRERCQLKcSJiIiIZCGFOBGRw5hZuZn9PzPbZ2b/E3Y9x8PMvmRm/x12HSKSOgpxIpKxzKzVzN4Ywku/E5gF1Lv7H4fw+iIir0ohTkTklRYAL7r76LE+0cyKUlCPiMgrKMSJSNYxs1Iz+7qZvRx8fd3MSoNtUTP7mZntNbMuM3vYzAqCbdeb2U4z6zWzjWZ2+STH/jvgb4F3mVmfmX3YzArM7AtmttXM2szsDjOrDvaPmZkH+20Dlh+h5j8ws7VBXY+Z2dlJ21rN7K/N7AUz6zaz75lZWdL2j5rZ5uD93Gdms5O2vcbMHgi27TGzzye9bElQa6+ZPW9mS0/skxeRTKIQJyLZ6G+AC4DFwDnAMuALwbbPATuABhJTop8H3MxOAz4FnO/uVcDvAa2HH9jdvwj8A/Ajd69091uBDwRflwJNQCXw74c99RLgjOC4hzCzc4HbgI8B9cB3gPsmgmfgfcFzTwZOnXg/ZnYZ8H+Bq4FGYCvww2BbFfAb4FfAbOAU4MGkY74t2LcGuG+SmkUkiynEiUg2eh/wZXdvc/d24O+Aa4JtIyTCzgJ3H3H3hz1xk+gxoBRYZGbF7t7q7i8dw+vd5O5b3L0P+Gvg3YdNnX7J3fvdff8kz78O+I67r3T3MXe/HRgiEUQn/Lu7b3f3LuDvgfckvfZt7r7G3YeC136tmcWAPwB2u/u/uPugu/e6+8qkYz7i7r9w9zHgv0gEXhHJEQpxIpKNZpM4IzVhazAG8E/AZuB+M9tiZjcAuPtm4DPAl4A2M/th8rTkcbxeEYkzfRO2H+X5C4DPBVOpe81sLzAvqebDn5/8fg557SBEdgJzgmMcLYjuTno8AJTpmj2R3KEQJyLZ6GUSwWjC/GCM4GzU59y9icR04mcnrn1z97vc/aLguQ587QRebxTYkzTmR3n+duDv3b0m6avC3X+QtM+8yd7P4a9tZhESU7I7g+M2TfE9iEiOUYgTkUxXbGZlSV9FwA+AL5hZg5lFSSxE+G84sIDgFDMzYB+JadRxMzvNzC4LrkMbBPYD41Os4QfAX5hZ3MwqOXjN3FRXr34X+LiZNVtCxMx+P7imbcInzWyumdWRuObvR0mv/UEzWxzU/g/ASndvBX4GNJrZZ4LFHlVm1jzFmkQkyynEiUim+wWJwDXx9SXgq8AqYB3wLLAmGANYSOJi/z7gceBb7r6CxPVwNwIdJKYZZ5K4vmwqbiNxTdnvgBYSIfDPpvoG3H0V8FESCwu6SUz3fuCw3e4C7ge2kJgi/Wrw3N8A/x9wD7CLxMKHdwfbeoE3AX8YvKdNJBZfiEgesMT1viIiEhYzawU+EgQ2EZEp0Zk4ERERkSykECciIiKShTSdKiIiIpKFdCZOREREJAspxImIiIhkobzr3B2NRj0Wi4VdhoiIiMirWr16dYe7N0y2Le9CXCwWY9WqVWGXISIiIvKqzGzrkbZpOlVEREQkCynEiYiIiGQhhTgRERGRLKQQJyIiIpKFFOJEREREspBCnIiIiEgWUogTERERyUIKcSIiIiJZSCFOREREJAspxImIiIhkIYU4ERERkSyUd/dOTbVNe3rZuXc/hQWW+DI7+LjAKDCjqPDQ8UnHCoyiiW3BmJmF/fZEREQkQyjETbO7ntzG9x5tTcmxzTgQ7CbCXrSylAX1FcTqI8SjERbUVxCPRphTU05RoU60ioiI5CqFuGn24Yvi/OE5sxkb90O/3BkbC74HY+PujB5hbNyd0Ymx8cTjibHxYHx03GnvHaKlo58nW7oYGB47UEdRgTGvroJYfQULFPBERERyjkLcNJtbW8Hc2oq0v6670943RGvHAK2d/bR29AffB1g5hYAXi0aI1Vco4ImIiGSJlIU4M5sH3AHMAhy42d3/zczqgB8BMaAVuNrdu5Oedz7wOPBud/9xMDYGPBvsss3d3xaMx4EfAvXAauAadx9O1XvKZGbGzKoyZlaVsSxed8i2QwLeRLibQsBLBLtEwDu5IRJKOBUREZHJmbun5sBmjUCju68xsyoSIesq4ANAl7vfaGY3ALXufn3wnELgAWAQuC0pxPW5e+Ukr3E3cK+7/9DM/hN4xt2/fbS6li5d6qtWrZq295nt3BNTsq2dhwa8lo4Btnb2HxLwvvfB87n0tJkhVisiIpJfzGy1uy+dbFvKzsS5+y5gV/C418zWA3OAK4E3BLvdDjwEXB/8/GfAPcD5r3Z8SyzVvAx4b9KxvgQcNcTJocyMmTPKmDnjCGfwgmvuPvC9p/jtxnaFOBERkQyRloufzCwGnAusBGYFAQ9gN4npVsxsDvB2Jg9hZWa2ysyeMLOrgrF6YK+7jwY/7yAREmWaTAS85qZ6liyoYWVLV9gliYiISCDlIc7MKkmcXfuMu/ckb/PEXO7EfO7XgevdfXySwywITiW+F/i6mZ18jDVcF4TAVe3t7cf8HgSWxerZsLuHfQMjYZciIiIipDjEmVkxiQB3p7vfGwzvCa6Xm7huri0YXwr80MxagXcC35o46+buO4PvW0hMv54LdAI1ZjYxJTwX2DlZHe5+s7svdfelDQ0N0/oe80VzUx3u8FSrzsaJiIhkgpSFuOCatVuB9e5+U9Km+4Brg8fXAj8FcPe4u8fcPQb8GPiEu/+vmdWaWWlwzChwIfBCcBZvBYnAd8ixZPotnldDSWEBTyrEiYiIZIRUnom7ELgGuMzM1gZfbwVuBN5kZpuANwY/H80ZwCoze4ZEaLvR3V8Itl0PfNbMNpO4Ru7WVLwRgbLiQhbPq2Hlls6wSxERERFSuzr1EeBIN/u8/FWe+4Gkx48BZx1hvy3AsuMsUY5Rc1Md33roJfqGRqksVZ9oERGRMKk1v0zZsngdY+PO6q3dr76ziIiIpJRCnEzZeQtqKSowTamKiIhkAIU4mbKKkiLOnFPNk+oXJyIiEjqFODkmzU11PLNjL/uTbsclIiIi6acQJ8ekOV7HyJjz9HZdFyciIhImhTg5JktjdRQYrNyiKVUREZEwKcTJMZlRVsyi2TN0XZyIiEjIFOLkmC2L1bNmWzdDo7ouTkREJCwKcXLMmpvqGBodZ92OfWGXIiIikrcU4uSYnR+rA9CUqoiISIgU4uSY1UVKOG1WFU+o6a+IiEhoFOLkuCyL17F6azejY+NhlyIiIpKXFOLkuDQ31TEwPMZzL/eEXYqIiEheUoiT47IsPnFdnKZURUREwqAQJ8dlZlUZTdGImv6KiIiERCFOjltzUx1PtnYxNu5hlyIiIpJ3FOLkuC2L19E7OMqG3bouTkREJN0U4uS4NcfrAd1HVUREJAwKcXLcZteUM7e2XE1/RUREQqAQJyekOV7Pk61duOu6OBERkXRSiJMT0txUR1f/MJvb+sIuRUREJK8oxMkJaQ76xT2hKVUREZG0UoiTEzK/roKTZpSxUvdRFRERSSuFODkhZsayeB1Ptui6OBERkXRSiJMT1txUR1vvEK2dA2GXIiIikjcU4uSENes+qiIiImmnECcn7OSGSqKVJWr6KyIikkYKcXLCJq6LW6kVqiIiImmjECfTYlmsjp1797OjW9fFiYiIpINCnEyL5ibdR1VERCSdFOJkWpw2q4rq8mLdR1VERCRNFOJkWhQUGOfH6lipFaoiIiJpoRAn06Y5Xkdr5wB7egbDLkVERCTnKcTJtGluSvSL0ypVERGR1FOIk2mzqHEGlaVFavorIiKSBgpxMm2KCgs4b0GtVqiKiIikgUKcTKvmpjo2tfXR2TcUdikiIiI5rSjsAiS3TNxH9anWLq44szHkakTy1+jYOC/u6ePZnXt5duc+igoKaGqI0BStpKkhQmN1GWYWdpkicgIU4mRanTWnhrLiAp7YohAnki7j405LZz/rduzlme37eHbnPp5/eR+DI+MAVJUWMe5O//DYgeeUFxcSj0YSwa6hkpODgBdviFBZql8NItlAf1NlWpUUFbBkfq2a/uYZd2d4bJzB4XH2j4wxODLG/uBrcHiMwdEx9gfb9o+MMTQyxv7hpH1GxhPPGR478PzBpG37R8aoKCmktqKEusjBr8TPxdRFSqmLFFNbUUJ9pJSqsiIKCnLzLJO7s3Pvftbt2MczO/aybvs+ntu5j96hUQDKigs4c3Y17122gHPmVXP23BoW1FVgBnt6htjS3sdLHf1sae+jpaOfdTv28YtndzHuB19j1ozSA2fsmhoqaQrC3tzaCgpz9HMVyUYKcTLtmuP1fP3BF9k3MEJ1RXHY5UgKuDs3/moDP16140AQc3/15x2uuNAoKyqkrKSQ8uJCyooLgu+F1EZKaCwqpLykkNKiAgaGx+geGGZPzyAbdvXQ2T/M0Oj4pMctLDBqK4oPCX21kRLqJvu5MvG9vKTwBD+V1GjrHeTZHft4Zsc+1u3Yy7M79tHZPwwkPr/TT5rB2xbP5py5NZw9r5pTGiopKpz8cueTqss4qbqM150SPWR8cGSMbV0DiYDX3s+W9n62dPTxs3W72Ld/5MB+JYUFLKivOCzcJc7i1VSUpO5DEJFJKcTJtGtuqsN/A6u2dnH5GbPCLkdS4I7Ht/Kd327h8tNn0tQQoSwIXuXFidCVHMaSx8uLCylN2lZ8hLAxFe7O/pExuvqH6e4fobN/iO6BYbr6R+juH6azf5ju/mG6BobZ3NaX2G9g+JAzTsnKiguoj5RSU1FMdXkxM8qKqSorYkbweEZ50eRj5cVUlkzPmb99AyOs27mXdUFgW7djH7v2JZpnFxgsnFnFpafP5Jy5iTNspzdWUVp04uGzrLiQU2dVceqsqkPG3Z2u/mG2BGfutrT381J7P5va+nhwfRujSR9mXaSEpmiEU2ZW8rbFs3ltU72uuRNJMYU4mXaL59VQUljAyhaFuFz06OYOvvyzF3jjGTO5+ZqloU1bmhkVJUVUlBQxt3Zqzxkfd3oGR+jqHz7wNRH8uvqHEgFwYJie/SO0dPTTMzhCz/6RQ64lm7wWqCwtCoJdEPSSgl8i9L1ybGB47EBYW7djL62dAweOGauv4PxYHWcHge01s2cQSfO1amZGfWUp9ZWlnB+rO2TbyNg427sGDpy12xKcwfv5s7v44VPbOf2kKj50UZy3nTObsuLMPMspku3Mj2cOJIstXbrUV61aFXYZOe/q/3ycodExfvqpi8IuRabRts4B3vYfj9BQWcq9n3gdVWX5MV0+OjZO7+AovYOjB4Jd4nvw8+DoIWO9h431Do4e9fiN1WUHwtrZc6s5e05N1l6KMDgyxn1rX+a2R1vYsLuX+kgJ77tgAX9ywXxmVpWFXZ5I1jGz1e6+dLJtOhMnKbEsXse3f/sSfUOjWumWI/qGRvnIHU/hDrdcuzRvAhwkGlnXBtfRHY/xcadvOAh1+w8GwaJC48w51TkVbsqKC7n6/Hn88dK5PP5SJ7c92sI3l2/i2w9t5g/Pmc2HLoxz5pzqsMsUyQkpa/ZrZvPMbIWZvWBmz5vZp4PxOjN7wMw2Bd9rD3ve+WY2ambvTBq7Nth/k5ldmzR+npk9a2abzewbpgswMkZzUx1j487qrd1hlyLTYHzc+YsfreWl9n7+471LWFAfCbukrFJQYMwoK2ZubQWLZs/ggqZ63vyak7js9Fk5FeCSmRmvOyXKLdeez/LPvYH3NS/gV8/t5g+++QhXf+dxfvXcbsaOdIGiiExJKu/YMAp8zt0XARcAnzSzRcANwIPuvhB4MPgZADMrBL4G3J80Vgd8EWgGlgFfTAp+3wY+CiwMvq5I4fuRY7Bkfi2FBab7qOaIr//mRR54YQ9/89YzuGhh9NWfIJIkHo3wpbe9hsf/+nK+8PtnsLN7Px//79Vc8k8ruOXhLfQMjrz6QUTkFVIW4tx9l7uvCR73AuuBOcCVwO3BbrcDVyU97c+Ae4C2pLHfAx5w9y537wYeAK4ws0Zghrs/4YkL++447FgSokhpEWfNqdZ9VHPAz9ft4hvLN/PH583lgxfGwi5Hslh1eTEfeX0Tv/3LN/Cff7KE2dXlfPXn63ntPzzIl+57ntaO/rBLFMkqablYycxiwLnASmCWu+8KNu0GZgX7zAHeDlwKnJ/09DnA9qSfdwRjc4LHh49P9vrXAdcBzJ8//8TejExZc1Mdtz3SwuDImFanZannX97H//mfZ1gyv4avvv1MtYyQaVFUWMAVZzZyxZmNPLtjH997tIU7V27l9sdbufz0mXzowjivPVktSkReTSqnUwEws0oSZ9c+4+49yduCM2gTF0V8Hbje3Sfv3nkC3P1md1/q7ksbGhqm+/ByBM3xOkbGnDXbdF1cNuroG+K6O1ZTU1HMf15z3rT0IxM53Flzq7npXYt59PrL+LNLT2HNtr2895aVvOXfHubup7YzOHL09i4i+SylIc7MikkEuDvd/d5geE8wFUrwfWLqdCnwQzNrBd4JfMvMrgJ2AvOSDjs3GNsZPD58XDLE0lgdZmhKNQsNj47zif9eQ0ffEDdfszRnL76XzDFzRhmfffNpPHbDZfzjO84G4K/uWceFNy7npvs30tYzGHKFIpknlatTDbgVWO/uNyVtug+YWGF6LfBTAHePu3vM3WPAj4FPuPv/Ar8G3mxmtcGChjcDvw6mZHvM7ILgtd4/cSzJDDPKilnUOEP3Uc1CX/p/z/Nkaxf/+M6zOWuu2kFI+ky0KPnlp1/PXR9p5tz5NXxzxWYu/NpyPvujtTy3c1/YJYpkjFReE3chcA3wrJmtDcY+D9wI3G1mHwa2Alcf7SDu3mVmXwGeCoa+7O4TqeATwPeBcuCXwZdkkOZ4PXeu3MrQ6Jim47LEfz2xlbtWbuPjl5zMlYsnvcxUJOUmWpS87pQorR39fP+xVv5n1XbufXony2J1fOiiGG9adBKFId0xRCQT6I4NklK/em43H//v1fz4469l6WG37ZHM88SWTv7klpVcfGoD333/Uv2ClIzSMzjC3U9t5/uPtbKjez+1FcXMriknWllKtLKUhqpSopUlNFSV0lBZSjT4XlNRrEUSkrV0xwYJzbJ4IritbOlSiMtw27sG+MSda1hQX8HX371YAU4yzoyyRIuSD14Y54EXdvPQxnbae4do7xti055e2vuGGBl75YmJogJLBL2qkkS4Swp4E98bqkqIVpZSXX5igW94dJz+oVH6hkbpHx6lbzB4PDRG39AIfUNj9A+N0j80Sm/wvX8ocUu3/uFRhkbGedOiWXzs4pOz9tZrkj4KcZJSdZESTptVxcqWLj55adjVyJH0D43y0TtWMTI2znffv5QZeXRLLck+hQV2oEVJMnenZ/8o7X1DtPcO0THZ974h1u/qpaNviNFJ7hhRUlhAfXA2L3GGL/G4sKBg0vDVF4SzREgbZXh0ag0WSosKqCorIlJaRKSkiMqyImZWlTEyNs63HnqJ/3piKx+7OBFYI7p1oRyB/mRIyi2L13Hvmh2Mjo1TVJjyrjZyjMbHnc/d/Qwv7unlex9cRlNDZdgliRwXM6O6opjqimJOmXn0P8fj486+/SMHAt7B4Dd8IPDt3jfIczv30dk/zNi4EykpJFKaCFyVQfiaU1NCVVkVkdLEtqrSIJiVJvapTH5cVkRlSREVpYUUH+X/hS+83MNND2zkn+9/ke892sonLz2F9zbPV79NeQWFOEm55qY6/uuJrTz3cg+L59WEXY4c5hvLN/Gr53fzhd8/g0tOVR9FyQ8FBUZtpITaSAkLZ1Uddd/x4IxdQZouMVg0ewa3XHs+a7Z188+/3siXf/YC3314C5++fCHvOG/uUQOg5Bf9SZCUm7guTvdRzTy/em4XX//NJv5oyRw+fFE87HJEMlJBgaUtwCVbMr+Wuz56AXd+pJlZM8q44d5nedNNv+Wna3ceCJaS3xTiJOVmVpXRFI2o6W+G2bC7h8/e/QyL59XwD28/S6v3RDLUhadE+cknXsct719KWXEhn/7hWt76jYd54IU95FuHCTmUQpykxbJ4HU+2djGmfz1mhK7+YT5y+yoqS4v4zjXn6VobkQxnZrxx0Sx+8eev59/evZjBkTE+escq3v6tx3h0c0fY5R1Ve+8QLR39OnuYAromTtKiuamOHz61nQ27e3jNbN0BIEwjY+N84s7VtPUOcffHXsusGbqllki2KCgwrlw8h7ee1cg9q3fwjQc38b5bVvLapnr+z++dxnkLasMukcGRMZ5q7eLhTR08vKmD9bsSt02vKCnk9JOqOKNxBotmz+CMxhmcflIVFSWKIsdLn5ykxbJ4PQBPtnQpxIXsKz97gSe2dHHT1edooYlIliouLODdy+Zz1blzuGvlNr710Gbe8e3HeOMZM/ncm0/jjMYZaavF3dm4p5eHX+zgd5vaebKli6HRcYoLjaUL6virK06jPlLC+l29vLCrh/vWvsydK7cBYAbx+ghnzJ7Bosbga/YMZlaV6hKPKVCIk7SYU1PO3NpyVm7p4oMX6gL6sNy1cht3PL6V6y5u4o+WzA27HBE5QWXFhXzoojjvOn8e33+sle/89iXe8m8P84fnzOYv3rgwZS2D2noHeXRzBw+/2MHDmzto7x0CYOHMSt7XvIDXnxqlOV436Vk2d2dH935e2NXD+l09vPByD+t27OXn63Yd2KcuUsKixhmc0Vh14KzdyQ2VWpl7GN12S9Lmc3c/w4qNbaz+whv1L6wQPNnSxXu/+wQXnhLltg+crzsyiOSgfQMj3PzwS9z2SCvDY+O8c8lc/vyNC5lTU35Cxz3SFGldpISLToly0cIor18YpbH6+F9n3/4RNkwEu109rN/Vy8Y9vQcaKJcUFnDqSZWccdLB6dgzGmdQXZ7bzcmPdtsthThJm7uf2s5f3bOOB/7i4lftyyTTa0f3AFf++6NUlxfzk09emPP/0xPJd+29Q3zroc3c+URi2vK9zfP55KWn0FBVOqXnuzsbdvfyyKZDp0hLCgs4b0Etrz81ysULG1jUOCOl7VdGxsbZ0t6fFOwSZ+46+4cP7DOnpvxAqFvUOIM5NeXUVZZQV1FCeUn2L9pSiEuiEBeerZ39XPJPD/GVq87kmgsWhF1O3hgYHuWd336c7V0D/O+nLuRk3ZFBJG/s3Lufbz64if9ZvYOSwgI+eGHsiPdlPdoU6esXNhx1ijSd3J323iGeDwLdRMBr6ejn8EhTXlxIXaSE2kgxdZFS6iqC75FXfq+tKKGmoiTjZikU4pIoxIXH3bng/z7Isng933zPuWGXkxfcnU/d9TS/eG4Xt117PpeePjPskkQkBC0d/fzrAy9y3zMvU1VWxMcubuK9zQt4/uV9R5wiff3CxDTpiUyRptPA8Cgv7umjrWeQ7oFhOvuH6e4/+L1rYISu/iG6+0foGxqd9BhmUFNeTF2k5JCv2oqSV4zVRUqYNaMs5dfpKcQlUYgL15//4Gme2NLJys9fruvi0uDfl2/in+9/kRvecjofv+TksMsRkZCt39XDv9z/Ir9Zv+fAWElhAUtjtVy0MD1TpJlgcGSMvQMjdPUPJ74GhunqG6JrYCQR+JK/BhIhcHSSPnf3/OnrUt7W5WghTqtTJa2Wxeu475mX2do5QCwaCbucnHb/87v55/tf5KrFs/nYxU1hlyMiGeCMxhnccu1S1mzr5rcb21k8vyYjpkjTray4kJOqCzmpemp9Mt2dnsHRw87sDdMU8u+x/PqvJqG7oClxH9WVLZ0KcSm0cXcvf/GjtZw9t5ob33G2znqKyCGWzK9lyfzwGwNnCzOjuryY6vLijPrdpYYrklYnN1RSHylhZYvuo5oq3f3DfPSOVVSUFnHzNUt1Sy0RkRylECdpZWYsi9excotCXCqMjo3zqR+sYfe+Qb5zzXlTnioQEZHsoxAnadccr2Pn3v3s6B4Iu5Scc++anTy6uZOvvv1MTZWIiOQ4hThJu+T7qMr0emD9HubUlPPH5+mWWiIiuU4hTtLu9JOqmFFWpCnVaTY0Osajmzu49PQGLWQQEckDCnGSdgUFievinmxViJtOK7d0MTA8xmVq6CsikhcU4iQUzfF6Wjr6aesZDLuUnLF8QxulRQW8tikadikiIpIGCnESiuYD/eJ0Nm46uDsrNrbxupPrc+KGzyIi8uoU4iQUixpnUFlaxMqWzrBLyQlbOvrZ2jmge6OKiOQRhTgJRVFhAectqNXihmmyYkMbAJeephAnIpIvFOIkNMvidWxq66OzbyjsUrLe8g1tLJxZyby6irBLERGRNFGIk9BM3Ef1Ka1SPSG9gyM82dKlVakiInlGIU5Cc9acGsqKC7S44QQ9urmD0XHX9XAiInlGIU5CU1JUwJL5ui7uRC3f0EZVWRHnLdBttkRE8olCnISqOV7P+t097Ns/EnYpWWl83FmxsZ2LT22guFB/nUVE8on+ry+hWhavwx1W6bq44/L8yz209w5xmValiojkHYU4CdW582soKdR1ccdr+YY2zOANpzWEXYqIiKSZQpyEqqy4kHPmVSvEHaflG9s4Z24N9ZWlYZciIiJpphAnoWuO1/Pczn30DY2GXUpW6egbYt2OvWrwKyKSpxTiJHTL4nWMjTtrtnaHXUpWeWhjO+6oP5yISJ5SiJPQnbeglsIC031Uj9GKDW00VJXymtkzwi5FRERCoBAnoYuUFnHWnGqe1HVxUzYyNs7vXmzn0tMaKCiwsMsREZEQKMRJRmiO1/HM9n0MjoyFXUpWWL21m96hUU2liojkMYU4yQjNTXUMj42zZpuui5uKFRvaKC40Llqo1iIiIvlKIU4ywnkL6jBDU6pTtHxDG8vidVSWFoVdioiIhEQhTjJCdXkxixpn6D6qU7C9a4BNbX1qLSIikucU4iRjLIvXsWZbN8Oj42GXktFWbGwD1FpERCTfKcRJxmiO1zM0Os66HXvDLiWjLd/QRqy+gqaGyrBLERGREKUsxJnZPDNbYWYvmNnzZvbpYLzOzB4ws03B99pg/EozW2dma81slZldlHSssWB8rZndlzQeN7OVZrbZzH5kZiWpej+SesvidQC6BddR7B8e4/GXOnmDplJFRPJeKs/EjQKfc/dFwAXAJ81sEXAD8KC7LwQeDH4meHyOuy8GPgTcknSs/e6+OPh6W9L414B/dfdTgG7gwyl8P5JidZESTp1VqRB3FI+91MHQ6LimUkVEJHUhzt13ufua4HEvsB6YA1wJ3B7sdjtwVbBPn7t7MB4BnKMwMwMuA358+LEkezXH61nd2sXomK6Lm8zyDW1UlBTS3FQXdikiIhKytFwTZ2Yx4FxgJTDL3XcFm3YDs5L2e7uZbQB+TuJs3ISyYIr1CTO7KhirB/a6+8Rd03eQCImTvf51wfNXtbe3T9fbkhQ4c84M+ofHeHnvYNilZBx3Z8WGNi48JUppUWHY5YiISMhSHuLMrBK4B/iMu/ckbwvOvHnSzz9x99NJnFH7StKuC9x9KfBe4OtmdvKx1ODuN7v7Undf2tCg5qiZLFYfAWBLR1/IlWSejXt6eXnfoKZSRUQESHGIM7NiEgHuTne/NxjeY2aNwfZGoO3w57n774AmM4sGP+8Mvm8BHiJxVq8TqDGziW6nc4GdqXs3kg7xhkSIa+3oD7mSzLNiQ+IssvrDiYgIpHZ1qgG3Auvd/aakTfcB1waPrwV+Gux/SvAczGwJUAp0mlmtmZUG41HgQuCF4CzeCuCdhx9LsldDZSmRkkJaOwfCLiXjrNjQxqLGGZxUXRZ2KSIikgFSeSbuQuAa4LKk9iBvBW4E3mRmm4A3Bj8DvAN4zszWAv8BvCsIamcAq8zsGRKh7UZ3fyF4zvXAZ81sM4lr5G5N4fuRNDAzYtEIW3Qm7hD7BkZYva1bU6kiInJAym686O6PAHaEzZdPsv/XSLQMOXz8MeCsI7zGFmDZCZQpGSgejbBux76wy8gov93Uzti4c6lCnIiIBHTHBsk48WiEHd0Duv1WkhUb2qitKGbxvJqwSxERkQyhECcZJ1YfYdxhW5euiwMYG3ce2tjGJac2UFhwpJPbIiKSbxTiJONoheqh1m7fS/fAiKZSRUTkEApxknHiQa+41k6FOEhMpRYYXHKqehyKiMhBCnGScWojJVSXF2uFamD5hjbOW1BLTUVJ2KWIiEgGUYiTjBSPRjSdCuzeN8gLu3o0lSoiIq+gECcZSSEu4aGNiRuaqD+ciIgcTiFOMlKsPsLL+wbZPzwWdimhWr6hjdnVZZw2qyrsUkREJMMoxElGmlihurUrf8/GDY2O8cjmDi49fSbBHelEREQOUIiTjHRghWoeT6k+2dLFwPCYplJFRGRSCnGSkWLRCgBaOvK34e/yDW2UFBXwupOjYZciIiIZSCFOMlJVWTHRylJaOvrCLiU0Kza08dqmespLCsMuRUREMpBCnGSseLSC1jw9E7elvY/WzgFNpYqIyBEpxEnGitVHaMnTuzYs36DWIiIicnQKcZKx4g0R2nuH6B0cCbuUtFuxsY1TZlYyr64i7FJERCRDKcRJxppYobq1M7+mVPuGRnmypUtn4URE5KgU4iRjxaKJENeSZ21GHtnUwciYc+lpCnEiInJkCnGSsWL1+RniVmxoo6qsiKWx2rBLERGRDKYQJxmrvKSQxuqyvGr46+6s2NjGxQsbKC7UX08RETky/ZaQjJZvK1Sff7mHtt4hLtX1cCIi8iqmFOLMLGJmBcHjU83sbWZWnNrSRBIrVPNpOnWitcgbTmsIuRIREcl0Uz0T9zugzMzmAPcD1wDfT1VRIhPi9RH2Doywd2A47FLSYvmGNs6ZW020sjTsUkREJMNNNcSZuw8AfwR8y93/GHhN6soSScinFaqdfUM8s2OvplJFRGRKphzizOy1wPuAnwdjuqGjpFw8j0LcQxvbcdddGkREZGqmGuI+A/w18BN3f97MmoAVKatKJDC/roICIy9WqC7f2Ea0spQzZ1eHXYqIiGSBoqns5O6/BX4LECxw6HD3P09lYSIAJUUFzKktpyXH79owMjbO715s54rXnERBgYVdjoiIZIGprk69y8xmmFkEeA54wcz+MrWliSTEo5W0dPSFXUZKrd7aTe/gqKZSRURkyqY6nbrI3XuAq4BfAnESK1RFUi5eX0FrxwDuHnYpKbNiYxvFhcZFC6NhlyIiIlliqiGuOOgLdxVwn7uPALn7G1UySiwaoW9olI6+3G0zsmJDG+fH6qgqU/tFERGZmqmGuO8ArUAE+J2ZLQB6UlWUSLJcX6G6o3uAF/f0aSpVRESOyZRCnLt/w93nuPtbPWErcGmKaxMBDoa4XF2huiK4S4P6w4mIyLGY6sKGajO7ycxWBV//QuKsnEjKzakpp6jAcvYeqss3tDG/roKmqP5KiYjI1E11OvU2oBe4OvjqAb6XqqJEkhUVFjC/voKW9twLcfuHx3jspU4uO30mZmotIiIiUzelPnHAye7+jqSf/87M1qagHpFJxesjtObgmbjHt3QwNDquqVQRETlmUz0Tt9/MLpr4wcwuBPanpiSRV4pFEyFufDy3FkUv39BGeXEhzfG6sEsREZEsM9UzcR8H7jCzifsBdQPXpqYkkVeKRyMMjoyzu2eQ2TXlYZczLdydFRvaufCUKGXFuhWxiIgcm6muTn3G3c8BzgbOdvdzgctSWplIklxcofrinj527t2v1iIiInJcpjqdCoC79wR3bgD4bArqEZlUbKJXXA5dF7di40RrkYaQKxERkWx0TCHuMFpKJ2nTOKOM0qKCnFqhunxDG2c0zqCxOjemh0VEJL1OJMTl1hXmktEKCoxYDq1Q3Tcwwuqt3Vyms3AiInKcjrqwwcx6mTysGaDTB5JWsWgFm9v6wi5jWvxuUztj467r4URE5LgdNcS5e1W6ChF5NfFoJcs3tDE6Nk5R4YmcRA7fig1t1FQUs3hebdiliIhIlsru34SSV+LRCkbGnJf3DoZdygkZG3ceerGdS05toLBAl5aKiMjxUYiTrBGrz40Vqs/s2EtX/7CmUkVE5ISkLMSZ2TwzW2FmL5jZ82b26WC8zsweMLNNwffaYPxKM1tnZmvNbNVhd4i4Nth/k5ldmzR+npk9a2abzewbpptP5rR4QxDi2rP7urgVG9ooMLjkVC1qEBGR45fKM3GjwOfcfRFwAfBJM1sE3AA86O4LgQeDnwken+Pui4EPAbdAIvQBXwSagWXAFyeCH/Bt4KPAwuDrihS+HwlZQ2UpkZJCWjsHwi7lhCzf0MaS+bXUVJSEXYqIiGSxlIU4d9/l7muCx73AemAOcCVwe7Db7cBVwT597j6xEjbCwVWxvwc84O5d7t4NPABcYWaNwAx3fyJ43h0Tx5LcZGbEohFasviuDXt6Bnn+5R7d8F5ERE5YWq6JM7MYcC6wEpjl7ruCTbuBWUn7vd3MNgA/J3E2DhLBb3vS4XYEY3OCx4ePSw6LZ3mIW7EhcZcGXQ8nIiInKuUhzswqgXuAzyTdsguA4AyaJ/38E3c/ncQZta9MYw3XBdfZrWpvb5+uw0oI4tEIO7oHGB4dD7uU47JiYxuN1WWcfpK694iIyIlJaYgzs2ISAe5Od783GN4TTIUSfG87/Hnu/jugycyiwE5gXtLmucHYzuDx4eOv4O43u/tSd1/a0KCLybNZrD7CuMP27uy7Lm5odIxHNnVw6ekz0RocERE5UalcnWrArcB6d78padN9wMQK02uBnwb7nzKxutTMlgClQCfwa+DNZlYbLGh4M/DrYEq2x8wuCJ73/oljSe46uEI1+6ZUn2rppn94jMtO01SqiIicuKPeseEEXQhcAzxrZmuDsc8DNwJ3m9mHga3A1cG2dwDvN7MRYD/wrmC6tcvMvgI8Fez3ZXfvCh5/Avg+iVuA/TL4khwWD3rFZeM9VJdvaKOkqIDXnVIfdikiIpIDUhbi3P0REvdYnczlk+z/NeBrRzjWbcBtk4yvAs48gTIly9RGSqguL87KxQ0rNrZxQVM9FSWp/LeTiIjkC92xQbJONq5Qbenop6Wjn8tO0zWZIiIyPRTiJOvEoxFasyzELT/QWmTWq+wpIiIyNQpxknVi9RFe3jfI4MhY2KVM2YoNbZzcEGF+fUXYpYiISI5QiJOsM7FCNVsWNwwMj7KypVMNfkVEZFopxEnWObBCNUumVJ/Zvo+RMee1J2tVqoiITB+FOMk6sWhiSrKlIzsa/q7Z1g3AufNqQ65ERERyiUKcZJ2qsmKilaW0dPSFXcqUPL2tm6ZohNpISdiliIhIDlGIk6wUj1bQmgVn4tydNdv2cu58nYUTEZHppRAnWSlWH6ElCxY2bO0coKt/mCULasIuRUREcoxCnGSleEOE9t4hegdHwi7lqCauh1uiM3EiIjLNFOIkK02sUN3amdlTqk9v20ukpJBTZ1WFXYqIiOQYhTjJSrFoIsRl+u231mzr5px5NRQWHOk2wiIiIsdHIU6yUqw+80PcwPAoG3b3aipVRERSQiFOslJ5SSGN1WUZ3fD3me37GBt3LWoQEZGUUIiTrJXpK1TV5FdERFJJIU6yVrwhktHTqWryKyIiqaQQJ1krXh9h78AIeweGwy7lFdTkV0REUk0hTrJWJq9Q3dalJr8iIpJaCnGSteJBiGvNwOvidD2ciIikmkKcZK35dRUUGLS0Z2CI25po8nvaSWryKyIiqaEQJ1mrpKiAObXltGTgXRvU5FdERFJNIU6yWjxamXG94tTkV0RE0kEhTrJavL6Clo5+3D3sUg5Qk18REUkHhTjJarFohL6hUTr6MqfNiBY1iIhIOijESVbLxBWqT2/bqya/IiKScgpxktUmQlymrFB1d57e1q0mvyIiknIKcZLV5tSUU1RgGXMP1W1dA3T2D3Pu/JqwSxERkRynECdZraiwgPn1FRmzQnXiejitTBURkVRTiJOsF6+PZMytt9TkV0RE0kUhTrJeLBqhtbOf8fHw24yoya+IiKSLQpxkvXg0wuDIOHt6B0OtQ01+RUQknRTiJOtlygrVdTvU5FdERNJHIU6yXmwixIW8QlVNfkVEJJ0U4iTrNc4oo7SoIPQVqmu2qsmviIikj0KcZL2CAiMW8grViSa/i9UfTkRE0kQhTnJCLFoRaoibaPKrRQ0iIpIuCnGSE+LRSrZ1DTAWUpsRNfkVEZF0U4iTnBCPVjAy5uzs3h/K66vJr4iIpJtCnOSEWH24K1TV5FdERNJNIU5yQrwhEeLCWKGqJr8iIhIGhTjJCQ2VpURKCkNZ3KAmvyIiEgaFOMkJZkYsGk6bkYlFDYvV5FdERNJIIU5yRjwaoTWEa+LWbN1LPBqhTk1+RUQkjRTiJGfEoxG2dw0wPDqettecaPJ7rpr8iohIminESc6I1UcYd9jePZC211STXxERCUvKQpyZzTOzFWb2gpk9b2afDsbrzOwBM9sUfK8Nxt9nZuvM7Fkze8zMzkk6VmswvtbMViWNT3osyU9hrFBVk18REQlLKs/EjQKfc/dFwAXAJ81sEXAD8KC7LwQeDH4GaAEucfezgK8ANx92vEvdfbG7L00aO9KxJA/FJ3rFpTHEPb1NTX5FRCQcKQtx7r7L3dcEj3uB9cAc4Erg9mC324Grgn0ec/fuYPwJYO4UXmbSY0l+qo2UUF1enNYQpya/IiISlrRcE2dmMeBcYCUwy913BZt2A7MmecqHgV8m/ezA/Wa22syuSxqfyrEws+vMbJWZrWpvbz+BdyKZLp0rVAeGR1m/S01+RUQkHCkPcWZWCdwDfMbde5K3ubuTCGjJ+19KIsRdnzR8kbsvAd5CYlr24sNfZ7JjJW272d2XuvvShoaGE3o/ktni0Qgt7ekJcRNNfrUyVUREwpDSEGdmxSQC3J3ufm8wvMfMGoPtjUBb0v5nA7cAV7p758S4u+8MvrcBPwGWvdqxJD/F6iO8vG+QwZGxlL/WxKKGc3UmTkREQpDK1akG3Aqsd/ebkjbdB1wbPL4W+Gmw/3zgXuAad38x6TgRM6uaeAy8GXjuaMeS/DWxQnVrZ+rbjKjJr4iIhKkohce+ELgGeNbM1gZjnwduBO42sw8DW4Grg21/C9QD30rkP0aDlaizgJ8EY0XAXe7+q+A5RzqW5KmDK1T7UrpidKLJ7yWnaXpeRETCkbIQ5+6PAEdasnf5JPt/BPjIJONbgHMOHw+2dU52LMlfsWgFAC0dqT0Tt71rv5r8iohIqHTHBskpVWXFRCtLU97wV01+RUQkbApxknPi0YqU94pbs61bTX5FRCRUCnGSc2L1EVpS3CtOTX5FRCRsCnGSc+INEdp7h+gbGk3J8Sea/Ko/nIiIhEkhTnLOxArVVF0XN9HkV9fDiYhImBTiJOfEohNtRlIT4tTkV0REMoFCnOScWIrPxD29TU1+RUQkfApxknPKSwpprC5LyZm4iSa/uh5ORETCphAnOSlVK1S3d+2no09NfkVEJHwKcZKT4g2RlEynqsmviIhkCoU4yUnx+gjdAyPsHRie1uOu2dZNhZr8iohIBlCIk5yUqhWqa7Z1c85cNfkVEZHwKcRJTooHIa51Gq+Lm2jyu2RBzbQdU0RE5HgpxElOml9XQYFBS/v0hbhn1eRXREQyiEKc5KSSogLm1JbT0jkwbcdcs20voCa/IiKSGRTiJGfFo5XTukJ1zbZuNfkVEZGMoRAnOSteX0FLRz/ufsLHUpNfERHJNApxkrNi0Qh9Q6N09J14mxE1+RURkUyjECc5azpXqB686X3NCR9LRERkOijESc6aCHHTsUL1QJPfWWryKyIimUEhTnLWnJpyigpsWu6hOtHkt6hQf2VERCQz6DeS5KyiwgLm11ec8ArV/cNjavIrIiIZRyFOclq8PnLCt95at2OvmvyKiEjGUYiTnBaLRmjt7Gd8/PjbjKjJr4iIZCKFOMlp8WiEwZFx9vQOHvcx1ORXREQykUKc5LQDK1SPc0pVTX5FRCRTKcRJToudYIibaPKrqVQREck0CnGS0xpnlFFaVHDcK1Qnmvwu0Zk4ERHJMApxktMKCoxYfYSWjoHjev7TavIrIiIZSiFOcl4sWkFLR99xPXfNtr1q8isiIhlJv5kk58WjlWzv2s/YMbYZSTT57VGTXxERyUgKcZLz4tEKhsfGeXnv/mN63rodexlVk18REclQCnGS82L1iRWqW45xcYOa/IqISCZTiJOcF29IhLhjXaGqJr8iIpLJFOIk5zVUlhIpKTymXnEHmvzOq0ldYSIiIidAIU5ynpkRi0aOKcTt6A6a/C7QVKqIiGQmhTjJC7FohNbOqYc4NfkVEZFMpxAneaEpGmFH936GR8entP+arWryKyIimU0hTvJCrD7C2LizvXtqd25Qk18REcl0+g0leSEWnfoKVTX5FRGRbKAQJ3mhKQhxU1ncoCa/IiKSDRTiJC/URkqoLi+eUoibaPK7WO1FREQkgynESd6Y6grVp7d1E6uvoL6yNA1ViYiIHB+FOMkbTdEIrR1HX9jg7qzZtldTqSIikvFSFuLMbJ6ZrTCzF8zseTP7dDBeZ2YPmNmm4HttMP4+M1tnZs+a2WNmdk7Ssa4ws41mttnMbkgaj5vZymD8R2am+yPJEcXqI+zcu5/BkbEj7pNo8jukJr8iIpLxUnkmbhT4nLsvAi4APmlmi4AbgAfdfSHwYPAzQAtwibufBXwFuBnAzAqB/wDeAiwC3hMcB+BrwL+6+ylAN/DhFL4fyXKxaAUAWzuPfDZOTX5FRCRbpCzEufsud18TPO4F1gNzgCuB24PdbgeuCvZ5zN27g/EngLnB42XAZnff4u7DwA+BK83MgMuAHx9+LJHJNEUrgaOvUFWTXxERyRZpuSbOzGLAucBKYJa77wo27QZmTfKUDwO/DB7PAbYnbdsRjNUDe9199LBxkUlNnIk7aohTk18REckSKf9NZWaVwD3AZ9y9J3mbuzvgh+1/KYkQd/001nCdma0ys1Xt7e3TdVjJMlVlxUQrS47Y8FdNfkVEJJukNMSZWTGJAHenu98bDO8xs8ZgeyPQlrT/2cAtwJXu3hkM7wTmJR12bjDWCdSYWdFh46/g7je7+1J3X9rQ0DA9b06yUjwaoeUIbUae3bmP0XHn3Hla1CAiIpkvlatTDbgVWO/uNyVtug+4Nnh8LfDTYP/5wL3ANe7+YtL+TwELg5WoJcC7gfuCs3grgHcefiyRI4nVR444nTqxqOFcLWoQEZEskMozcRcC1wCXmdna4OutwI3Am8xsE/DG4GeAvyVxndu3gn1XAQTXvH0K+DWJxRF3u/vzwXOuBz5rZpuD596awvcjOSAWjdDeO0Tf0Ogrtq3Zqia/IiKSPYpefZfj4+6PAHaEzZdPsv9HgI8c4Vi/AH4xyfgWEqtXRaZk4h6qrR39nDmn+sD4RJPfixdGwypNRETkmGgJnuSVWBDiDp9SVZNfERHJNgpxkldi9QfPxCVTk18REck2CnGSV8pLCmmsLnvFClU1+RURkWyjECd5Z7IVqmryKyIi2Ua/sSTvxKKRQ6ZTJ5r8qrWIiIhkE4U4yTtN0QjdAyPsHRgGDjb5XTJfixpERCR7KMRJ3jl8haqa/IqISDZSiJO8E49WANAaLG5Qk18REclGCnGSd+bVVVBg0NIxcKDJr6ZSRUQk2yjESd4pLSpkTm05LR39avIrIiJZSyFO8lKsPrFCVU1+RUQkWynESV5qCtqMqMmviIhkK4U4yUuxaITeoVF+s76Ns+dWq8mviIhkHf3mkrw00WZk5979WtQgIiJZSSFO8lJTEOIAhTgREclKCnGSl+bUlFNUYICa/IqISHYqCrsAkTAUFRYwv66CcXc1+RURkaykECd56+NvOPnA2TgREZFsoxAneevqpfPCLkFEROS46Zo4ERERkSykECciIiKShRTiRERERLKQQpyIiIhIFlKIExEREclCCnEiIiIiWUghTkRERCQLKcSJiIiIZCGFOBEREZEspBAnIiIikoUU4kRERESykEKciIiISBZSiBMRERHJQubuYdeQVmbWDmxN8ctEgY4Uv0a20GdxkD6LBH0OB+mzOEifxUH6LBL0OSQscPeGyTbkXYhLBzNb5e5Lw64jE+izOEifRYI+h4P0WRykz+IgfRYJ+hxenaZTRURERLKQQpyIiIhIFlKIS42bwy4gg+izOEifRYI+h4P0WRykz+IgfRYJ+hxeha6JExEREclCOhMnIiIikoUU4qaZmV1hZhvNbLOZ3RB2PWEws3lmtsLMXjCz583s02HXFCYzazWzZ81srZmtCruesJjZacFnMPHVY2afCbuudDGz28yszcyeSxqrM7MHzGxT8L02zBrT5QifxT+Z2QYzW2dmPzGzmhBLTIsjfA5fMrOdSX9P3hpmjelyhM/iR0mfQ6uZrQ2xxIyk6dRpZGaFwIvAm4AdwFPAe9z9hVALSzMzawQa3X2NmVUBq4Gr8u1zmGBmrcBSd1e/o0Dwd2Un0Ozuqe7bmBHM7GKgD7jD3c8Mxv4R6HL3G4N/9NW6+/Vh1pkOR/gs3gwsd/dRM/saQK5/Fkf4HL4E9Ln7P4dZW7pN9lkctv1fgH3u/uW0F5fBdCZuei0DNrv7FncfBn4IXBlyTWnn7rvcfU3wuBdYD8wJtyrJMJcDL+VLgANw998BXYcNXwncHjy+HbgqnTWFZbLPwt3vd/fR4McngLlpLyzNjvBnIi8d7bMwMwOuBn6Q1qKygELc9JoDbE/6eQd5Hl7MLAacC6wMuZQwOXC/ma02s+vCLiZDvBv9DxlglrvvCh7vBmaFWUwG+RDwy7CLCNGngmnl2/Jliv1VvB7Y4+6bwi4k0yjEScqYWSVwD/AZd+8Ju54QXeTuS4C3AJ8Mpg3ylpmVAG8D/ifsWjKJJ65tyfvrW8zsb4BR4M6wawnJt4GTgcXALuBfQq0mM7wH/aNvUgpx02snMC/p57nBWN4xs2ISAe5Od7837HrC5O47g+9twE9ITLvns7cAa9x9T9iFZIA9wTWkE9eStoVcT6jM7APAHwDv8zy9YNvd97j7mLuPA98lz/9/YWZFwB8BPwq7lkykEDe9ngIWmlk8ONvwbuC+kGtKu+D6hVuB9e5+U9j1hMnMIsHiDswsArwZeO7oz8p5+lf1QfcB1waPrwV+GmItoTKzK4C/At7m7gNh1xOWiVAfeDv6/8UbgQ3uviPsQjKRVqdOs2A5+NeBQuA2d//7cCtKPzO7CHgYeBYYD4Y/7+6/CK+qcJhZE4mzbwBFwF35+GdiQhBktwFN7r4v7HrSycx+ALwBiAJ7gC8C/wvcDcwHtgJXu3vOX+h+hM/ir4FSoDPY7Ql3/3goBabJET6HN5CYSnWgFfhY0nWTOWuyz8LdbzWz75P4s/CfIZaXsRTiRERERLKQplNFREREspBCnIiIiEgWUogTERERyUIKcSIiIiJZSCFOREREJAspxImIJDGzMTNbm/R1wzQeO2Zm+d73S0SmSVHYBYiIZJj97r447CJERF6NzsSJiEyBmbWa2T+a2bNm9qSZnRKMx8xseXDD8gfNbH4wPsvMfmJmzwRfrwsOVWhm3zWz583sfjMrD+1NiUhWU4gTETlU+WHTqe9K2rbP3c8C/p3EnVkAvgnc7u5nk7hp+zeC8W8Av3X3c4AlwPPB+ELgP9z9NcBe4B0pfTcikrN0xwYRkSRm1ufulZOMtwKXufsWMysGdrt7vZl1AI3uPhKM73L3qJm1A3PdfSjpGDHgAXdfGPx8PVDs7l9Nw1sTkRyjM3EiIlPnR3h8LIaSHo+ha5NF5DgpxImITN27kr4/Hjx+DHh38Ph9wMPB4weBPwUws0Izq05XkSKSH/QvQBGRQ5Wb2dqkn3/l7hNtRmrNbB2Js2nvCcb+DPiemf0l0A58MBj/NHCzmX2YxBm3PwV2pbp4EckfuiZORGQKgmvilrp7R9i1iIiAplNFREREspLOxImIiIhkIZ2JExEREclCCnEiIiIiWUghTkRERCQLKcSJiIiIZCGFOBEREZEspBAnIiIikoX+f1vSR9fTCLEiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_epochs = 20\n",
    "fig, axes = plt.subplots(1, 1, figsize=(10, 6))\n",
    "metrics_df = pd.DataFrame({'Loss' : gpt2_losses, 'Epoch' : [idx for idx in range(num_epochs)]})\n",
    "axes.xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))\n",
    "sns.lineplot(data=metrics_df, y='Loss', x='Epoch', ax=axes)\n",
    "axes.set_title('Loss for epoch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "del trained_model\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Need to git into fine-fitting GPT2 deeper. Results are not good for now."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN-based network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpu = torch.device(\"cpu\")\n",
    "device = torch.device(\"cuda\")\n",
    "dtype = torch.float32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNModel(nn.Module):\n",
    "    def __init__(self, dataset):\n",
    "        super().__init__()\n",
    "        self.lstm_size = 128\n",
    "        self.embedding_dim = 128\n",
    "        self.num_layers = 3\n",
    "\n",
    "        n_vocab = len(dataset.uniq_words)\n",
    "        self.embedding = nn.Embedding(\n",
    "            num_embeddings=n_vocab,\n",
    "            embedding_dim=self.embedding_dim,\n",
    "        )\n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size=self.lstm_size,\n",
    "            hidden_size=self.lstm_size,\n",
    "            num_layers=self.num_layers,\n",
    "            dropout=0.2,\n",
    "        )\n",
    "        self.fc = nn.Linear(self.lstm_size, n_vocab)\n",
    "\n",
    "    def forward(self, x, prev_state):\n",
    "        embed = self.embedding(x)\n",
    "        output, state = self.lstm(embed, prev_state)\n",
    "        logits = self.fc(output)\n",
    "        return logits, state\n",
    "\n",
    "    def init_state(self, sequence_length):\n",
    "        return (torch.zeros(self.num_layers, sequence_length, self.lstm_size, dtype=dtype, device=device),\n",
    "                torch.zeros(self.num_layers, sequence_length, self.lstm_size, dtype=dtype, device=device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(\n",
    "        self,\n",
    "        data,\n",
    "        sequence_length\n",
    "    ):\n",
    "        self.train_df = data\n",
    "        self.sequence_length = sequence_length\n",
    "        self.words = self.load_words()\n",
    "        self.uniq_words = self.get_uniq_words()\n",
    "        \n",
    "        self.index_to_word = {index: word for index, word in enumerate(self.uniq_words)}\n",
    "        self.word_to_index = {word: index for index, word in enumerate(self.uniq_words)}\n",
    "\n",
    "        self.words_indexes = [self.word_to_index[w] for w in self.words]\n",
    "\n",
    "    def load_words(self):\n",
    "        text = self.train_df['text'].str.cat(sep=' ')\n",
    "        return text.split(' ')\n",
    "\n",
    "    def get_uniq_words(self):\n",
    "        word_counts = Counter(self.words)\n",
    "        return sorted(word_counts, key=word_counts.get, reverse=True)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.words_indexes) - self.sequence_length\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return (\n",
    "            torch.tensor(self.words_indexes[index:index+self.sequence_length], dtype=torch.long, device=device),\n",
    "            torch.tensor(self.words_indexes[index+1:index+self.sequence_length+1], dtype=torch.long, device=device),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(dataset, model, batch_size, sequence_length, max_epochs):\n",
    "    model.train()\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = AdamW(model.parameters(), lr=0.001)\n",
    "    losses = []\n",
    "    progress_bar_epoch = tqdm(range(max_epochs))\n",
    "    for epoch in range(max_epochs):\n",
    "        state_h, state_c = model.init_state(sequence_length)\n",
    "\n",
    "        for batch, (x, y) in enumerate(dataloader):\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            y_pred, (state_h, state_c) = model(x, (state_h, state_c))\n",
    "            loss = criterion(y_pred.transpose(1, 2), y)\n",
    "\n",
    "            state_h = state_h.detach()\n",
    "            state_c = state_c.detach()\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        progress_bar_epoch.update(1)\n",
    "        \n",
    "        # Eval\n",
    "        with torch.no_grad():\n",
    "            local_losses = []\n",
    "            for batch, (x, y) in enumerate(dataloader):\n",
    "                y_pred, (state_h, state_c) = model(x, (state_h, state_c))\n",
    "                loss = criterion(y_pred.transpose(1, 2), y).detach().item()\n",
    "                state_h = state_h.detach()\n",
    "                state_c = state_c.detach()\n",
    "                local_losses.append(loss)\n",
    "            losses.append(sum(local_losses) / len(local_losses))\n",
    "            print('Loss:', losses[-1])\n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()\n",
    "    return losses\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(dataset, model, text, next_words=10, sentences_count=10):\n",
    "    model.eval()\n",
    "\n",
    "    sentences = []\n",
    "    for sentence_number in range(sentences_count):\n",
    "        words = text.split(' ')\n",
    "        state_h, state_c = model.init_state(len(words))        \n",
    "        for i in range(0, next_words):\n",
    "            x = torch.tensor([[dataset.word_to_index[w] for w in words[i:]]], device=device)\n",
    "            y_pred, (state_h, state_c) = model(x, (state_h, state_c))\n",
    "\n",
    "            last_word_logits = y_pred[0][-1]\n",
    "            p = torch.nn.functional.softmax(last_word_logits, dim=0).detach().cpu().numpy()\n",
    "            word_index = np.random.choice(len(last_word_logits), p=p)\n",
    "            words.append(dataset.index_to_word[word_index])\n",
    "        sentences.append(' '.join(words))\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e0d0900625647388f3b6ddc7e0c7f45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 7.022284302009544\n",
      "Loss: 6.809778588677421\n",
      "Loss: 6.629134669521738\n",
      "Loss: 6.513660735890345\n",
      "Loss: 6.42966286543057\n",
      "Loss: 6.373067394731009\n",
      "Loss: 6.315290740298741\n",
      "Loss: 6.247657624598082\n",
      "Loss: 6.161425924543197\n",
      "Loss: 6.101483059413542\n",
      "Loss: 6.047153238112552\n",
      "Loss: 5.980212228552339\n",
      "Loss: 5.937773416490119\n",
      "Loss: 5.96207544404238\n",
      "Loss: 5.8754516041218325\n",
      "Loss: 5.847995290296332\n",
      "Loss: 5.815798257208113\n",
      "Loss: 5.791129127371735\n",
      "Loss: 5.718700500914288\n",
      "Loss: 5.692583308002066\n",
      "Loss: 5.674070111386062\n",
      "Loss: 5.653235460901018\n",
      "Loss: 5.626087978406606\n",
      "Loss: 5.6209270857312355\n",
      "Loss: 5.619382070405834\n",
      "Loss: 5.641821304553656\n",
      "Loss: 5.566472973315244\n",
      "Loss: 5.537402569945089\n",
      "Loss: 5.458705756264894\n",
      "Loss: 5.470052897022461\n",
      "Loss: 5.456377739833696\n",
      "Loss: 5.454246841711441\n",
      "Loss: 5.445400017167106\n",
      "Loss: 5.44966743561217\n",
      "Loss: 5.441196030166549\n",
      "Loss: 5.475529805052704\n",
      "Loss: 5.486502378725159\n",
      "Loss: 5.464949283503034\n",
      "Loss: 5.455759821809488\n",
      "Loss: 5.423226798246355\n",
      "Loss: 5.475510696469224\n",
      "Loss: 5.473018140357158\n",
      "Loss: 5.297500553469972\n",
      "Loss: 5.194313068075228\n",
      "Loss: 5.205753052295162\n",
      "Loss: 5.2559645829467\n",
      "Loss: 5.183197025115115\n",
      "Loss: 5.210912805523364\n",
      "Loss: 5.191279550494277\n",
      "Loss: 5.111647770489533\n"
     ]
    }
   ],
   "source": [
    "batch_size = 512\n",
    "sequence_length = 7\n",
    "max_epochs = 50\n",
    "\n",
    "dataset = Dataset(subsamples_data, sequence_length)\n",
    "model = RNNModel(dataset).to(device)\n",
    "rnnmodel_losses = train(dataset, model, batch_size, sequence_length, max_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in the mcdonald's  ho snort wack streiht lactose. hate in on\n",
      "in the worry thirsty travel test day boot inside you at? everybody\n",
      "in the cards didn't rob dogg or real man they still down?\n"
     ]
    }
   ],
   "source": [
    "gc.collect()\n",
    "torch.cuda.empty_cache()\n",
    "generated_sentences = predict(dataset, model, text=generation_start, sentences_count=3)\n",
    "for sentence in generated_sentences:\n",
    "    print(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Loss for epoch')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAGDCAYAAABuj7cYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA/r0lEQVR4nO3dd3jV9d3/8ec7exASQkKAhCkb2REUcFcUtziqdVVtqdbaebe1e9x31323/WltXXVWwaogVlsHVq2jIBD23jNAEkZCIDt5//7IwUZkBMjJ9yR5Pa7rXOec7zjndfq9Gl98vsvcHRERERGJDFFBBxARERGR/1A5ExEREYkgKmciIiIiEUTlTERERCSCqJyJiIiIRBCVMxEREZEIonImIm2KmSWa2atmVmJmLwad50SY2U/N7Nmgc4hIeKiciUggzGyTmX0mgK++BsgCOrr7tQF8v4jIUamciUhb0wNY4+41x7uimcWEIY+IyCeonIlIRDGzeDO7z8y2hx73mVl8aF6Gmf3dzIrNbI+ZfWBmUaF53zWzfDMrNbPVZnb+YT77Z8CPgc+a2X4zu8PMoszsh2a22cwKzewvZpYaWr6nmXlouS3AO0fIfKmZLQrlmmVmQxvM22Rm3zOzFWa218yeNLOEBvO/aGbrQr/nFTPr2mDeYDN7KzSvwMy+3+Br40JZS81suZnlntz/8iISKVTORCTS/AA4HRgODANGAz8MzfsWsA3IpH7X5PcBN7P+wFeA09w9BbgQ2HToB7v7T4BfAs+7ezt3fxz4fOhxLtAbaAf88ZBVzwYGhj73E8xsBPAE8CWgI/AI8MrBQhlyY2jdU4B+B3+PmZ0H/Aq4DugCbAb+GpqXAvwTeAPoCvQB3m7wmZeHlk0DXjlMZhFpoVTORCTS3Aj83N0L3b0I+Blwc2heNfUlpoe7V7v7B15/g+BaIB4YZGax7r7J3dcfx/f93t03uPt+4HvA9Yfswvypux9w9/LDrD8ZeMTd57h7rbs/DVRSXzAP+qO7b3X3PcAvgBsafPcT7r7A3StD332GmfUELgV2uvvv3L3C3UvdfU6Dz/zQ3V9z91rgGeqLrIi0AipnIhJpulI/gnTQ5tA0gP8D1gEzzWyDmd0L4O7rgK8DPwUKzeyvDXcPnsD3xVA/MnfQ1qOs3wP4VmiXZrGZFQPdGmQ+dP2Gv+cT3x0qh7uB7NBnHK1g7mzwugxI0DFxIq2DypmIRJrt1Beeg7qHphEaPfqWu/emfrfeNw8eW+buU919fGhdB35zEt9XAxQ0mOZHWX8r8At3T2vwSHL35xos0+1wv+fQ7zazZOp3jeaHPrd3I3+DiLQiKmciEqRYM0to8IgBngN+aGaZZpZB/QH8z8LHB973MTMDSqjfnVlnZv3N7LzQcV4VQDlQ18gMzwHfMLNeZtaO/xyT1tizOf8M3GlmY6xespldEjpm7KC7zSzHzNKpP6bu+QbffZuZDQ9l/yUwx903AX8HupjZ10MnSaSY2ZhGZhKRFkzlTESC9Br1Rerg46fA/wB5wBJgKbAgNA2gL/UHye8HZgMPuvu71B9v9mtgF/W7+zpRf/xWYzxB/TFb7wMbqS939zT2B7h7HvBF6g/I30v9btfPH7LYVGAmsIH6XZX/E1r3n8CPgOnADupPGLg+NK8UuAC4LPSb1lJ/0oKItHJWfyytiIiEg5ltAr4QKmIiIsekkTMRERGRCKJyJiIiIhJBtFtTREREJIJo5ExEREQkgqiciYiIiESQVnU16YyMDO/Zs2fQMURERESOaf78+bvcPfPQ6a2qnPXs2ZO8vLygY4iIiIgck5ltPtx07dYUERERiSAqZyIiIiIRROVMREREJIKonImIiIhEEJUzERERkQiiciYiIiISQVTORERERCKIypmIiIhIBFE5ExEREYkgKmciIiIiEUTlTERERCSCqJwdh+XbS1i4ZW/QMURERKQVC1s5M7P+ZraowWOfmX39kGXMzP5gZuvMbImZjWww71YzWxt63BqunMfja39dxG9nrg46hoiIiLRiMeH6YHdfDQwHMLNoIB+YcchiE4G+occY4CFgjJmlAz8BcgEH5pvZK+4e6LDVhEFZPPL+BkrKqklNig0yioiIiLRSzbVb83xgvbtvPmT6FcBfvN5HQJqZdQEuBN5y9z2hQvYWcFEzZT2iCYM7U1vnvL2qIOgoIiIi0ko1Vzm7HnjuMNOzga0N3m8LTTvS9EANzU6lc/sEZi5XORMREZHwCHs5M7M44HLgxTB9/mQzyzOzvKKionB8xceioowLBmXx3poiKqprw/pdIiIi0jY1x8jZRGCBux9uuCkf6NbgfU5o2pGmf4q7P+ruue6em5mZ2USRj2zC4CzKq2v5YO2usH+XiIiItD3NUc5u4PC7NAFeAW4JnbV5OlDi7juAN4EJZtbBzDoAE0LTAnd6746kJMQwc/nOoKOIiIhIKxS2szUBzCwZuAD4UoNpdwK4+8PAa8DFwDqgDLgtNG+Pmf03MC+02s/dfU84szZWbHQU5w/oxD9XFlBTW0dMtC4VJyIiIk0nrOXM3Q8AHQ+Z9nCD1w7cfYR1nwCeCGe+E3Xh4M68vGg7eZv3cnrvjsdeQURERKSRNOxzAs7ql0lcTJTO2hQREZEmp3J2ApLjYzizTwZvLt9J/eCfiIiISNNQOTtBFw7uTH5xOSt27As6ioiIiLQiKmcn6PyBnYgytGtTREREmpTK2Qnq2C6e3B7pvKlLaoiIiEgTUjk7CRMGZ7FqZylbdpcFHUVERERaCZWzkzBhUGcAZq7Q6JmIiIg0DZWzk9C9YxIDOqfouDMRERFpMipnJ+nCwZ3J27yHXfsrg44iIiIirYDK2UmaMDiLOod3VhYGHUVERERaAZWzkzSoS3uy0xJ11qaIiIg0CZWzk2RmXDi4Mx+s28WBypqg44iIiEgLp3LWBCYMzqKqpo731xQFHUVERERaOJWzJpDbowMdkmK1a1NEREROmspZE4iJjuIzA7N4e1Uh1bV1QccRERGRFkzlrIlMGNyZ0ooa5mzYE3QUERERacFUzprImX0zSIyN1q5NEREROSkqZ00kITaas/tl8taKAurqPOg4IiIi0kKpnDWhCYOz2LmvgqX5JUFHERERkRZK5awJnT8gi+go065NEREROWEqZ00oNSmW03unM3OFboQuIiIiJ0blrIlNGNSZdYX7WV+0P+goIiIi0gKpnDWxCwZlATBzuUbPRERE5PipnDWxrmmJDM1JZeYKHXcmIiIix0/lLAwuHNyZhVuKyS8uDzqKiIiItDAqZ2Fw+bCuALy8MD/gJCIiItLSqJyFQbf0JMb0Smf6/G2464K0IiIi0ngqZ2Fy9cgcNuw6wKKtxUFHERERkRZE5SxMJg7pTEJsFNMXbAs6ioiIiLQgKmdhkpIQy4WDO/Pq4h1U1tQGHUdERERaiLCWMzNLM7NpZrbKzFaa2RmHzP+2mS0KPZaZWa2ZpYfmbTKzpaF5eeHMGS5Xj8yhpLyad1YWBh1FREREWohwj5zdD7zh7gOAYcDKhjPd/f/cfbi7Dwe+B7zn7nsaLHJuaH5umHOGxbg+GWS1j9euTREREWm0sJUzM0sFzgIeB3D3KncvPsoqNwDPhStPEKKjjCtHZPOv1UXs2l8ZdBwRERFpAcI5ctYLKAKeNLOFZvaYmSUfbkEzSwIuAqY3mOzATDObb2aTw5gzrK4emUNNnfPKou1BRxEREZEWIJzlLAYYCTzk7iOAA8C9R1j2MuDfh+zSHO/uI4GJwN1mdtbhVjSzyWaWZ2Z5RUVFTRi/afTLSmFIdiovLdSuTRERETm2cJazbcA2d58Tej+N+rJ2ONdzyC5Nd88PPRcCM4DRh1vR3R9191x3z83MzGyS4E1t0shsluXvY/XO0qCjiIiISIQLWzlz953AVjPrH5p0PrDi0OVCx6adDfytwbRkM0s5+BqYACwLV9Zwu3xYV2KijJd0YoCIiIgcQ7jP1rwHmGJmS4DhwC/N7E4zu7PBMlcBM939QINpWcCHZrYYmAv8w93fCHPWsOnYLp5zB3RixsJ8amrrgo4jIiIiESwmnB/u7ouAQy+D8fAhyzwFPHXItA3UX3qj1bh6ZDZvrSjgw3W7OKd/p6DjiIiISITSHQKaybkDOpGWFMtLC/KDjiIiIiIRTOWsmcTHRHPZ0K68uXwn+yqqg44jIiIiEUrlrBldPSqHypo6Xl+6I+goIiIiEqFUzprRsJxUemcmM32+dm2KiIjI4amcNSMz4+qROczdtIctu8uCjiMiIiIRSOWsmV01IhszdMcAEREROSyVs2bWNS2Rsad05KUF+bh70HFEREQkwqicBWDSiBy27Ckjb/PeoKOIiIhIhFE5C8BFp3YmKS6a6fO1a1NEREQ+SeUsAMnxMUw8tQv/WLKDiuraoOOIiIhIBFE5C8jVI7Mpraxh5oqCoKOIiIhIBFE5C8jpvTvSNTWBlxZo16aIiIj8h8pZQKKijKtGZvP+miIK91UEHUdEREQihMpZgK4emUOdw5OzNgUdRURERCKEylmAeme2Y9KIbB7/cKPuGCAiIiKAylngvnPRAGKijF++tjLoKCIiIhIBVM4C1jk1gbvP7cMby3cya92uoOOIiIhIwFTOIsAd43uR0yGRn/99BTW1dUHHERERkQCpnEWAhNhofnjJQFbtLOW5eVuDjiMiIiIBUjmLEBcO7swZvTvy+5mrKSmrDjqOiIiIBETlLEKYGT++bBAl5dXc9/aaoOOIiIhIQFTOIsjALu25YXR3/jJ7M2sLSoOOIyIiIgFQOYsw37ygH8lx0fz87ytw96DjiIiISDNTOYswHdvF8/XP9OODtbt4Z1Vh0HFERESkmamcRaCbz+jBKZnJ/PffV1BVo0triIiItCUqZxEoNjqKH182mE27y3hq1sag44iIiEgzUjmLUGf3y+T8AZ34w9vrKCqtDDqOiIiINBOVswj2g0sGUllTy2/fXB10FBEREWkmKmcRrHdmOz4/ticvzN/KsvySoOOIiIhIM1A5i3D3nN+X9KQ4fvbqcl1aQ0REpA0IazkzszQzm2Zmq8xspZmdccj8c8ysxMwWhR4/bjDvIjNbbWbrzOzecOaMZO0TYvn2hf2Zt2kvry7ZEXQcERERCbNwj5zdD7zh7gOAYcDKwyzzgbsPDz1+DmBm0cCfgInAIOAGMxsU5qwR69rcbpya3Z6fvrKc/OLyoOOIiIhIGIWtnJlZKnAW8DiAu1e5e3EjVx8NrHP3De5eBfwVuCIsQVuA6CjjD9ePoLqmjjufmU9FdW3QkURERCRMwjly1gsoAp40s4Vm9piZJR9muTPMbLGZvW5mg0PTsoGtDZbZFprWZvXObMd91w9naX4JP3x5mY4/ExERaaXCWc5igJHAQ+4+AjgAHHrs2AKgh7sPAx4AXj7eLzGzyWaWZ2Z5RUVFJxk5sp0/MIuvnd+XafO38exHm4OOIyIiImEQznK2Ddjm7nNC76dRX9Y+5u773H1/6PVrQKyZZQD5QLcGi+aEpn2Kuz/q7rnunpuZmdnUvyHifO38vpw/oBM/e3UF8zbtCTqOiIiINLGwlTN33wlsNbP+oUnnAysaLmNmnc3MQq9Hh/LsBuYBfc2sl5nFAdcDr4Qra0sSFWX8/rPD6ZaexJenLKBgX0XQkURERKQJhftszXuAKWa2BBgO/NLM7jSzO0PzrwGWmdli4A/A9V6vBvgK8Cb1Z3i+4O7Lw5y1xUhNjOWRm0dxoLKGu56dT2WNThAQERFpLaw1HViem5vreXl5QcdoNq8t3cGXpyzgc2O688urhgQdR0RERI6Dmc1399xDp+sOAS3YxUO6cNc5pzB1zhaen7cl6DgiIiLSBFTOWrj/mtCfM/tm8KOXl7Noa3HQcUREROQkqZy1cAcvUNupfTx3PjOfotLKoCOJiIjISVA5awU6JMfxyM2jKC6v4u6pC6iurQs6koiIiJwglbNWYnDXVH5z9VDmbtzDL/5xuFuYioiISEsQE3QAaTpXDM9mybYSHv9wI+7Ojy4dREy0+reIiEhLonLWynz/4oFERxmPvr+BzXvKeOCGEaQkxAYdS0RERBpJwyqtTHSU8f2LB/KrSUP4cO0urnloNtv2lgUdS0RERBpJ5ayVumF0d56+fTTbS8q58k//ZuGWvUFHEhERkUZQOWvFxvXJYMaXx5EUF8P1j37E35dsDzqSiIiIHIPKWSvXp1M7Xr57HENzUvnK1IU88PZaWtMtu0RERFoblbM2ID05jme/MIarRmTzu7fW8K0XFutm6SIiIhFKZ2u2EfEx0fz+umH0zkjmd2+tYdvech6+eRTpyXFBRxMREZEGNHLWhpgZ95zflz9+bgSLtxVz1YP/Zn3R/qBjiYiISAMqZ23QpUO78tfJp3OgsoZJD85izobdQUcSERGREJWzNmpE9w7M+PI4MlPiufnxuby8MD/oSCIiIoLKWZvWLT2J6XeOZWSPNL7+/CKdySkiIhIBVM7auNSkWP5y+xgmhc7k/M60JVTX1gUdS0REpM3S2ZpCXEwUv7tuGN3Sk7j/7bXsKKngwZtG0l735BQREWl2GjkToP5Mzm9c0I/fXjuMjzbs5pqHZumenCIiIgFQOZNPuGZUDn+5fTQ7Siq46sFZLN1WEnQkERGRNkXlTD5lbJ8MXrprLHHRUVz3yGz+uaIg6EgiIiJthsqZHFbfrBRm3D2WvlntmPxMHs98tDnoSCIiIm2CypkcUaeUBP46+XTOG9CJH728jKdnbQo6koiISKunciZHlRQXw0M3jWLCoCx+8spyjaCJiIiEmcqZHFNsdBR//NxIPjOwfgRt6pwtQUcSERFptVTOpFHiYqL4040jObd/Jt+fsZTn56mgiYiIhIPKmTRafEw0D900irP7ZXLvS0t5MW9r0JFERERaHZUzOS4JsdE8cvMoxvfJ4DvTl/DSgm1BRxIREWlVVM7kuCXERvPozbmc0bsj//XiYv62KD/oSCIiIq1GWMuZmaWZ2TQzW2VmK83sjEPm32hmS8xsqZnNMrNhDeZtCk1fZGZ54cwpxy8xLprHbz2N0b3S+cbzi3h18fagI4mIiLQK4R45ux94w90HAMOAlYfM3wic7e5DgP8GHj1k/rnuPtzdc8OcU05AYlw0T3z+NHJ7pPP15xfx2tIdQUcSERFp8cJWzswsFTgLeBzA3avcvbjhMu4+y933ht5+BOSEK4+ER1JcDE/edhojuqXx1ecW8saynUFHEhERadHCOXLWCygCnjSzhWb2mJklH2X5O4DXG7x3YKaZzTezyUdaycwmm1memeUVFRU1TXI5LsnxMTx1+2iG5qTylakLmLtxT9CRREREWqxwlrMYYCTwkLuPAA4A9x5uQTM7l/py9t0Gk8e7+0hgInC3mZ11uHXd/VF3z3X33MzMzCb9AdJ47eJjePK20eR0SOQrUxdQWFoRdCQREZEWKZzlbBuwzd3nhN5Po76sfYKZDQUeA65w990Hp7t7fui5EJgBjA5jVmkCqYmxPHTTKPZVVHPP1IXU1NYFHUlERKTFCVs5c/edwFYz6x+adD6wouEyZtYdeAm42d3XNJiebGYpB18DE4Bl4coqTWdgl/b88qohzNm4h/+buTroOCIiIi1OTJg//x5gipnFARuA28zsTgB3fxj4MdAReNDMAGpCZ2ZmATNC02KAqe7+RpizShOZNDKHBVv28sh7GxjRrQMXndo56EgiIiIthrl70BmaTG5urufl6ZJokaCyppbrHp7NhqIDvHLPeHplHO1cEBERkbbHzOYf7nJhukOAhEV8TDQP3jSKmGjjrmfnU15VG3QkERGRFkHlTMImOy2R+64fweqCUn4wYymtaZRWREQkXFTOJKzO7pfJ18/vx0sL85k6d0vQcURERCKeypmE3T3n9eGc/pn87JUVLN5aHHQcERGRiKZyJmEXFWX8v+uGk5kSz5enLGDvgaqgI4mIiEQslTNpFh2S43joppEUlVbytecXUVun489EREQOR+VMms3QnDR+evlg3l9TxAPvrA06joiISERSOZNmdcPoblw9Mof7317LG8t2Bh1HREQk4qicSbMyM/7nylMZmpPG3VMX8Pw8ncEpIiLSkMqZNLvEuGimfmEM4/pk8N3pS3ng7bW6BpqIiEiIypkEIjk+hsdvzWXSiGx+99YafvS3ZTpJQEREhPDf+FzkiGKjo/jddcPo1D6Bh99bz67SKu67fjgJsdFBRxMREQmMRs4kUGbGvRMH8ONLB/Hmip3c8vhcSsqqg44lIiISGJUziQi3j+/FH64fwaKtxVz7yCx2lJQHHUlERCQQjSpnZpZsZlGh1/3M7HIziw1vNGlrLhvWladuO43txRVMenAWawtKg44kIiLS7Bo7cvY+kGBm2cBM4GbgqXCFkrZrbJ8Mnv/S6dTUOdc8PJu8TXuCjiQiItKsGlvOzN3LgEnAg+5+LTA4fLGkLRvcNZWX7hpLx+Q4bnxsDm8u18VqRUSk7Wh0OTOzM4AbgX+EpumUOgmbbulJTLtrLAO7tOfOZ+fzp3fX6VpoIiLSJjS2nH0d+B4ww92Xm1lv4N2wpRIB0pPjeO6Lp3P5sK7835uruee5hZRX1QYdS0REJKwadZ0zd38PeA8gdGLALnf/ajiDiUD93QTu++xwBnZpz2/eWMXGXQd49JZcstMSg44mIiISFo09W3OqmbU3s2RgGbDCzL4d3mgi9cyMO88+hSduPY0tu8u4/IEPmbtRJwqIiEjr1NjdmoPcfR9wJfA60Iv6MzZFms25Azrx8lfGkZoYy+f+/BFT5mwOOpKIiEiTa2w5iw1d1+xK4BV3rwZ0dLY0u1My2zHj7nGM75vBD2Ys44cvL6W6ti7oWCIiIk2mseXsEWATkAy8b2Y9gH3hCiVyNKmJsTx+62l86ezePPvRFm58bA6791cGHUtERKRJ2IlensDMYty9ponznJTc3FzPy8sLOoY0o78tyuc705aQ0S6eR28ZxeCuqUFHEhERaRQzm+/uuYdOb+wJAalm9nszyws9fkf9KJpIoK4Yns20O8dS5841D83mw7W7go4kIiJyUhq7W/MJoBS4LvTYBzwZrlAix2NITip/+8o4enRM4o6n56mgiYhIi9bYcnaKu//E3TeEHj8DeoczmMjx6JSSwNQvnk6vjGTueHoe/16ngiYiIi1TY8tZuZmNP/jGzMYB5eGJJHJi0pPjmPKFMR8XtFkqaCIi0gI1tpzdCfzJzDaZ2Sbgj8CXwpZK5AR1bBfPlC+MoUd6MreroImISAvUqHLm7ovdfRgwFBjq7iOA8461npmlmdk0M1tlZitDN09vON/M7A9mts7MlpjZyAbzbjWztaHHrcf5u6QN69gunilfHEP39KT6grZeBU1ERFqOxo6cAeDu+0J3CgD4ZiNWuR94w90HAMOAlYfMnwj0DT0mAw8BmFk68BNgDDAa+ImZdTierNK2ZbSLZ+oXT68vaE+poImISMtxXOXsEHbUmWapwFnA4wDuXuXuxYcsdgXwF6/3EZBmZl2AC4G33H2Pu+8F3gIuOoms0gYdWtBmr98ddCQREZFjOplydqyr1/YCioAnzWyhmT0WunF6Q9nA1gbvt4WmHWn6p5jZ5IPXXysqKjquHyCt38GC1q1DfUH7aIMKmoiIRLajljMzKzWzfYd5lAJdj/HZMcBI4KHQMWoHgHubJvZ/uPuj7p7r7rmZmZlN/fHSChwsaDkdErntSRU0ERGJbEctZ+6e4u7tD/NIcfeYY3z2NmCbu88JvZ9GfVlrKB/o1uB9TmjakaaLnJDMlPqClh0qaO+uKgw6koiIyGGdzG7No3L3ncBWM+sfmnQ+sOKQxV4BbgmdtXk6UOLuO4A3gQlm1iF0IsCE0DSRE1Zf0MbQo2MStz01j288v0g3TBcRkYhzrNGvk3UPMMXM4oANwG1mdieAuz8MvAZcDKwDyoDbQvP2mNl/A/NCn/Nzd98T5qzSBnRKSeDlu8fx4LvreOi99fxrdSE/vGQQk0ZmY3bUc1xERESahbkf67j+liM3N9fz8vKCjiEtxJqCUu6dvoQFW4o5s28Gv7hyCN07Jh3XZ+zaX8kri7YzZ+NuvjdxID0zDj3nRURE5PDMbL67535qusqZtGV1dc6zczbzv2+spqaujm98ph93jO9FTPSR9/hXVNfy1ooCZizM5701RdTWOTFRRp9O7Zjx5XEkxkU34y8QEZGWSuVM5Ch2lJTzo5eX88+VBQzu2p7fXD2UU7NTP55fV+fM3bSHGQvyeW3pDkora+iSmsAVw7OZNDKb7cXl3PbUPCaNyOG31w7VLlIRETmmI5WzcB9zJtIidElN5M+3jOKNZTv58SvLufyPH3LH+F5cOSKb15bu4OWF28kvLic5LpqLTu3CpJHZnN67I9FR9SWsX1YK95zXlz+8vZbcnh24YXT3gH+RiIi0VBo5EzlESXk1v359Fc/N3QJAlMGZfTOZNDKbCwZlkRR3+H/T1NY5n39yLnM27GH6XWMZkpN62OVERERAuzVFjtv8zXtYtbOUCwZm0al9QqPW2XOgikv/8AFRUcbf7xlPWlJcmFOKiEhLdaRyFrbrnIm0dKN6pHPjmB6NLmYA6clx/OnGkRTsq+CbLyymrq71/ONHRESah8qZSBMb0b0DP7xkEO+sKuSh99YHHUdERFoYlTORMLjljB5cNqwrv5u5mn+v2xV0HBERaUFUzkTCwMz49aQh9M5sx1efW8jOkoqgI4mISAuhciYSJsnxMTx800jKq2v5ytQFVNfWBR1JRERaAJUzkTDq0ymF31w9lLzNe/n166uCjiMiIi2AyplImF02rCufH9uTxz/cyGtLdwQdR0REIpzKmUgz+P7FAxnRPY3vTFvCqp37go4jIiIRTOVMpBnExUTxp8+NJD4mion3f8DtT83j3VWF1Oo6aCIicgjdIUCkGRXsq2DKR5t5bt5WikoryemQyOfGdOe63G5ktIsPOp6IiDQj3b5JJIJU19Yxc3kBz360mdkbdhMXHcXEIZ256fQe5PbogJkFHVFERMJM5UwkQq0rLOXZj7YwfcE2Sitq6J+Vwk1n9OCqEdm0iz/8TdZFRKTlUzkTiXBlVTW8smg7z3y0meXb95ESH8MtY3tw+7hedNQuTxGRVkflTKSFcHcWbS3msQ828tqyHSTERPO5Md2ZfFZvso7jJuwiIhLZVM5EWqB1haU8+K/1/G3RdqLNuO60HL501il0S08KOpqIiJwklTORFmzL7jIeem890+ZvxR2uHJHNl885hd6Z7YKOJiIiJ0jlTKQV2FFSzqPvb+C5uVuorKnjkiFduPvcPgzs0j7oaCIicpxUzkRakV37K3n8w408M3sz+ytrOK1nB64emcPFQ7vQPiE26HgiItIIKmcirVBJWTVT525h2vytrC86QHxMFBMGd+bqkdmM75NBTLRuAiIiEqlUzkRaMXdnybYSpi/YxiuLt1NcVk1mSjxXjchm0shsBnTWbk8RkUijcibSRlTW1PLuqkKmL8jn3VWF1NQ5g7u25+qROVw9MofUJO32FBGJBCpnIm3Q7v2VvLJ4Oy8tyGdpfgm9M5J58c4zdFFbEZEIcKRypgNSRFqxju3iuW1cL169ZzxTvjCG/OJyPv/kPPZX1gQdTUREjkDlTKSNGNcng4duGsmKHfuY/Jc8Kqprg44kIiKHEdZyZmabzGypmS0ys0/tbzSzb4fmLTKzZWZWa2bpjVlXRI7feQOy+O21Q5m1fjdf++tCamrrgo4kIiKHaI6Rs3Pdffjh9qm6+/+F5g0Hvge85+57GrOuiJyYq0bk8ONLB/Hm8gJ+MGMZrem4UxGR1iAm6AAN3AA8F3QIkbbg9vG92FtWxQPvrKNDchz3ThwQdCQREQkJ98iZAzPNbL6ZTT7SQmaWBFwETD/edUXkxHzzgn7cOKY7D7+3nkffXx90HBERCQn3yNl4d883s07AW2a2yt3fP8xylwH/PmSXZqPWDRW3yQDdu3cPx28QaZXMjJ9fcSrF5dX88rVVpCXFcV1ut6BjiYi0eWEdOXP3/NBzITADGH2ERa/nkF2ajV3X3R9191x3z83MzGyq6CJtQnSU8f+uG86ZfTO4d/oS3ly+M+hIIiJtXtjKmZklm1nKwdfABGDZYZZLBc4G/na864rIyYuLieLhm0YxNCeNe55byOz1u4OOJCLSpoVz5CwL+NDMFgNzgX+4+xtmdqeZ3dlguauAme5+4FjrhjGrSJuWHB/Dk58/je7pSXzxL3ksyy8JOpKISJul2zeJyMd2lJRzzUOz2VdezTW5OVx/Wnf6d04JOpaISKuk2zeJyDF1SU3kuS+eztn9M5ny0RYuvO99rnrw3zw/bwsHdMsnEZFmoZEzETmsPQeqmLEwn7/O3cLawv0kx0Vz2bCufPa0bgzvloaZBR1RRKRFO9LImcqZiByVu7NgSzHPz9vCq4t3UF5dy4DOKXz2tG5cNSKbtKS4oCOKiLRIKmcictJKK6p5dfEOnp+3hcXbSoiLjmJM73TO7pfJWf0y6dupnUbUREQaSeVMRJrUiu37mL5gG/9aXcj6ovqTrbukJnBm3wzO6pfJ+D4ZGlUTETkKlTMRCZv84nLeX1PE+2uK+HDdLkoraogyGJqTxln9Mjm7XwbDctKIidY5SCIiB6mciUizqKmtY/G2Yt5bs4v31xSxeFsx7tAhKZbzB2ZxwaAszuqbSWJcdNBRRUQCpXImIoEoLqvig7W7eGdVIW+vLGBfRQ0JsVGc2TeTCYOyOH9gFunJ2v0pIm3PkcpZuG98LiJtXFpSHJcN68plw7pSXVvH3I17mLl8JzNXFPDWigKiDE7rmc6EwZ2ZMCiLbulJQUcWEQmURs5EJBDuzrL8fcxcsZOZywtYXVAKwOCu7bn+tG5cOSKblITYgFOKiISPdmuKSETbtOsAb60o4OVF+Szfvo+kuGiuGN6VG8f04NTs1KDjfUJtnRMdpUuGiMjJUTkTkRbB3VmyrYQpczbzyuLtVFTXMaxbGjeO6c5lQ7sGdiJBWVUNb68s5B9LdvDu6kJuGN2dn1w2SNd1E5ETpnImIi1OSXk1MxZs49k5W1hXuJ/2CTFcPSqHG8d0p0+n8N+QvbyqlndX1xeyt1cVUFFdR2ZKPH07tWPW+t1896IB3HXOKWHPISKtk8qZiLRY7s7cjXuYMmcLry/bQXWtM7pXOkOzU+mQHEeHpDjSk2NDz3F0SI4jLTH2hK6rVlFdy79WF/L3JTt4e2Uh5dW1ZLSL5+IhnblkSBdye6ZjwNeeX8Sri7dz32eHc+WI7Kb/0SLS6ulsTRFpscyMMb07MqZ3R3btH8S0+duYPn8bU+duoayq9ojrtU+IIT05jnYJMcRGRxEbHUVcdBQx0fbx69jQ69iYKErKqnl3dSFlVbV0TI5j0shsLh3aldG90j91jNlvrx1KUWkF3562mMyUeMb1yQj3/wwi0kZo5ExEWrSK6lqKy6rZc6CKvWVVn3w+UMXesmr2V9ZQXVsXejjVtXVU1fznfU1tHVW1Tmy0cU7/Tlw6tAtjeqUfc+StpLya6x6ezfbicl648wwGdmnfTL9aRFoD7dYUEQmD7cXlTHpwFgAvfXksXdMSA04kIi3FkcqZbnQnInISuqYl8tTtp3GgsobPPzmXkvLqoCOJSAunciYicpIGdG7PI7eMYuOuA0z+Sx6VNUc+Dk5E5FhUzkREmsDYUzL47bXDmLNxD996YTF1da3nkBERaV46W1NEpIlcMTybHSUV/Pr1VXRNS+T7Fw8MOpKItEAqZyIiTehLZ/VmR3E5j76/gS6pCdw2rlfQkUSkhVE5ExFpQmbGjy8bzM59Ffz87ytITYzlqhHZus2TiDSajjkTEWli0VHG/dePYFT3DnzzhcVMvP8Dnp+3hYpqnSggIsemciYiEgYJsdE8+4Ux/O81QwH47vSljP31O/z2zdUU7KsIOJ2IRDJdhFZEJMzcndkbdvPkvzfxz5UFRJtx6dAu3DauF8O6pQUdT0QContriogExMwYe0oGY0/JYPPuAzw9azMv5G3l5UXbGdWjA7eN68lFgzuf0I3aRaT10ciZiEgASiuqmTZ/G0/N2sTm3WVkpyXy9O2n0adTStDRRKSZ6PZNIiIRJCUhltvG9eKdb53Dn2/JpbKmjtuemsfu/ZVBRxORgKmciYgEKDrKuGBQFo/dmkvhvkq+9Mx83f5JpI0Lazkzs01mttTMFpnZp/Y3mtk5ZlYSmr/IzH7cYN5FZrbazNaZ2b3hzCkiErTh3dL4/XXDydu8l3unL6U1HXIiIsenOU4IONfddx1l/gfufmnDCWYWDfwJuADYBswzs1fcfUUYc4qIBOqSoV3YuKsfv525hl4ZyXz1/L5BRxKRAETqbs3RwDp33+DuVcBfgSsCziQiEnZ3n9uHSSOz+f1ba3h18fag44hIAMJdzhyYaWbzzWzyEZY5w8wWm9nrZjY4NC0b2NpgmW2haSIirZqZ8atJQxjdM51vvbiYBVv2Bh1JRJpZuMvZeHcfCUwE7jazsw6ZvwDo4e7DgAeAl4/3C8xsspnlmVleUVHRSQcWEQlafEw0D988ii6pCUz+Sx5b95QFHUlEmlGzXefMzH4K7Hf33x5lmU1ALtAX+Km7Xxia/j0Ad//V0b5D1zkTkdZkXeF+Jj34b7qkJjLtrjNISYgNOlKL5O5sL6lgbUEp6wr3s7ZgP2sLS9myp5xTs9sz8dTOXDCoM+nJcUFHlTbmSNc5C1s5M7NkIMrdS0Ov3wJ+7u5vNFimM1Dg7m5mo4FpQA8gGlgDnA/kA/OAz7n78qN9p8qZiLQ2/163i1ufmMu4Phk8fmuu7iJwDOVVtczesCtUwOof6wpKOVD1n8uTZLSLo0+ndnRNS2Tuxj1s21tOdJQxplc6E0/tzITBnclqnxDgr/iP4rIq1hXuJyE2msS46Prn2GgSYqNIiIkmKsqCjignIYhy1huYEXobA0x191+Y2Z0A7v6wmX0FuAuoAcqBb7r7rND6FwP3UV/UnnD3XxzrO1XORKQ1em7uFr730lJuPaMHP7vi1KMuW1fn5BeXs6aglIrqOk7Nbk/39CTMWvd/xCuqa3lu7hb+9O56doUu5NspJZ6+We3o2ynl4+c+ndp9YoTM3Vm+fR9vLNvJ68t2sL7oAGYwsnsHJp7amQsHd6ZbelIgv2nWul185bmF7DlQdcRl4mKiSAwVtqz28Tx9+2jSkjQC2FI0ezkLgsqZiLRWv/jHCv78wUZ+dvlgbh3bE3enYF8lqwtKWVtQyuqdpawpKGVt4X7Kqj55EdvUxFiG5qQyJDuVoTlpDM1JpUtqQqsobNW1dbyYt40H3lnLjpIKxvRK58vn9mF4ThqpSce/G3htQWmoqO1kxY59AJya3Z6xp2TQo2MSPdKT6dExiS6pCWEbxXR3Hnl/A//7xip6Z7bjOxf2B6C8upaK6loqqus+fl1eXUtFVS37KmqYsTCfn142iM+P6xWWXNL0VM5ERFqw2jrnS8/M551VBQzvlsbawv2UVtR8PD+jXTz9strRLyuF/p1T6JfVjviYaJbml7BkWwlLthWzemcpNXX+8fJDc1IZmpPKoC7tSU2MJT42mviYqPpHbDQJoef4mChioiyiylxNbR0vL9rO/W+vYeueckZ0T+O/JvRn7Ckdmyzn5t0HeHN5fVFbvn0fVTV1H8+LiTJyOiTSo2N9WeuenkSPjsn0z0qhe8cTH2krrajm2y8u4Y3lO7lkSBf+95qhJMc37pKkl/zhAwD+8dUzT/j7pXmpnImItHAHKmv41guL2XOgin6d64vYwUdjDmavqK5l5Y59LM0vYfHWEpbmF7O2cD+N+c9AlNWfRZqREkf39Poy0u3gc4f657Sk2GMWI3enorqOkvJq9lVUc6Cyhox28Y0eiaqrc/6xdAf/759r2FB0gMFd2/NfE/pzTv/MsJbHujpn574KNu8uY/PuA2zeU8aW3WVs3nOAzbvKKK38T1H+zMBOfOW8vgzvlnZc37GusJQvPTOfTbvL+N7EAdwxvtdx/aanZ23iJ68s57Wvnsmgru2P67slGCpnIiLyKQcqa+p3hVbWUFlTR2VNbf1zdR0VNbVUVv9nWkV1LYWllWzZU8bWPWXs2v/JY6FS4mM+LmwdkmMpragJlbAaSsurPy5k1bWf/u9OdJTRNS2BnLQkuqUnktMhiZwOiXRLr3/ulJLA2ysL+P1ba1i1s5R+We345gX9uXBwVuAjeu7O3rJqNu8+wPtrdvHEvzdSUl7NmX0z+Or5fTmtZ/oxP+O1pTv49ouLSYyL5oEbRnLGKR2PO0dxWRWjf/E2N57enZ9cNvjYK0jgVM5ERKRJHaisYeve+hGkg4Vty54ytu4tp7isivYJsaQkxtI+IYbUxFjaJ8bWPyeEnhNjSIqLpnBfJdv2lrN1b/1nbNtbTmFp5Se+KzrKqK1zemUk8/XP9OXSoV2JjtAzFfdX1vDM7M089sEGdh+o4vTe6Xz1vL6ccZhdrjW1dfzfm6t55P0NjOiexoM3jqRLauIJf/fdUxYwa/0u5nz/M8TF6MzeSKdyJiIiLUZFdS35xeVsDZW9bXvL6NcphSuGd20xlxMpq6ph6pwtPPr+BgpLKxnVowP3nNeHs/vV74Ldtb+Se6YuZPaG3dx8eg9+dOmgky5U764u5LYn5/HQjSOZOKRLE/0SCReVMxERkQBUVNfyYt5WHvrXeraXVDA0J5Vrc7vx4Lvr2HOgil9eNYSrR+U0yXfV1jnjfv0Og7q254nPn9Yknynhc6Ry1jL++SEiItJCJcRGc/MZPfnXt8/l15OGUFxWzY9eXkZMtPHSl8c2WTGD+t2/k0Zm86/VhRTsq2iyz5Xm1bjzc0VEROSkxMVEcf3o7lwzKocP1u1iZLcOJ3QttmO5ZlQOD/5rPS8tyOeuc05p8s+X8NPImYiISDOKiY7i3P6dwlLMAHpntuO0nh14cf5WWtOhS22JypmIiEgrc+2obmwoOsCCLXuDjiInQOVMRESklbl4aBcSY6N5MW9b0FHkBKiciYiItDLt4mO4ZGgX/r5kB2VVNcdeQSKKypmIiEgrdO2oHPZX1vDGsp1BR5HjpHImIiLSCo3ulU6Pjkm8kLc16ChynFTOREREWiEz45qROXy0YQ9bdpcFHUeOg8qZiIhIK3X1qBzMYNoCnRjQkqiciYiItFJd0xIZ3yeD6fO3UVena561FCpnIiIirdi1ud3ILy5n1vrdQUeRRlI5ExERacUmDMqifUIML87XiQEthcqZiIhIK5YQG80Vw7N5Y9lOSsqrg44jjaByJiIi0spdm5tDZU0dry7eHnQUaQSVMxERkVZuSHYqAzqn8OJ8nbXZEqiciYiItHJmxjWjcli8tZg1BaVBx5FjUDkTERFpA64akU1MlPGi7hgQ8VTORERE2oCO7eI5b0AnZizMp7q2Lug4chQqZyIiIm3Edbnd2LW/indWFQYdRY5C5UxERKSNOKd/Jt3SE/nN66uoqK4NOo4cgcqZiIhIGxETHcUvrhzChl0HeOCdtUHHkSNQORMREWlDzuqXyTWjcnj4vQ0s314SdBw5DJUzERGRNuaHlwykQ1Ic352+hBqdHBBxwlrOzGyTmS01s0VmlneY+Tea2ZLQMrPMbFhj1xUREZETk5YUx8+vGMyy/H089uHGoOPIIWKa4TvOdfddR5i3ETjb3fea2UTgUWBMI9cVERGREzTx1M5MGJTF/3trDRcO7kyvjOSwf2dxWRUpCbFER1nYv6slC3S3prvPcve9obcfATlB5hEREWkrzIz/vvJU4mKiuHf6EurqPGzf5e68MG8rp//qbT7/5Fwqa3Sm6NGEu5w5MNPM5pvZ5GMsewfw+gmuKyIiIscpq30CP7h4IHM27uGv88Jz54Cyqhq+9eJivjN9Cb0y2vHB2l1864XF1IaxDLZ04d6tOd7d882sE/CWma1y9/cPXcjMzqW+nI0/gXUnA5MBunfvHp5fISIi0kp99rRu/G3Rdn712krOG9CJzqkJTfbZawpK+fKUBawv2s83PtOPr5zXh8c+2MCvXl9Fh9Bxb2baxXmosI6cuXt+6LkQmAGMPnQZMxsKPAZc4e67j2fd0PxH3T3X3XMzMzOb/keIiIi0YmbGryYNobqujh++vAz3phnRejFvK5f/8UOKy6qZcscYvvaZvkRHGV86+xQmn9WbZz7azH3/1LXWDids5czMks0s5eBrYAKw7JBlugMvATe7+5rjWVdERESaRs+MZL55QT/+ubKAfyzdcVKfVV5Vy3+9uJhvT1vCiG4deO1r4xnbJ+MTy3xv4gCuGZXD/W+v5S+zN53U97VG4dytmQXMCA1XxgBT3f0NM7sTwN0fBn4MdAQeDC1X4+65R1o3jFlFRETatNvH9eLvS3bwk78tZ9wpGXRIjjvuz1hbUMrdUxewtnA/Xz2/L187v+9hz8w0M349aQjFZdX85JXlpCXFcfmwrk3xM1oFa6rhy0iQm5vreXm6JJqIiMiJWLljH5c98CGXD+/K768bflzrvrRgGz+YsYzk+Gju++wIxvfNOOY6FdW13PL4XBZu3cvjt57GWf2O7/Akd2d7SQVd2icQ1QIvz2Fm80ODUp/QHNc5ExERkRZgYJf23HXOKTzwzjouH9aVc/p3OuryJeXVrC0o5fl5W3lx/jbG9ErnDzeMIKt9404qSIiN5s+35nL9ox9x57PzmfKFMYzo3uGY6+2vrGHGwnyenb2Z1QWldEyO45z+nThvQCfO7JdB+4TYRn1/pNLImYiIiHyssqaWi+//gIrqOt78xlm0i4+hvKqWdYX7WV1QypqCUlbvrH/eUVIBgBncfU4fvv6ZvsREH//h7IWlFVzz0Gz2VVQz7c4z6NMp5bDLrSko5ZnZm3lpwTYOVNVyanZ7LhvalZU79vGvNUUUl1UTE2Wc1jOd8wZ04twBnTglMzlizwg90siZypmIiIh8Qt6mPVz7yGz6Z6VQUV3L5j1lHKwLcTFR9MlsR//OKfTLSqF/53YM6pJ60pfg2Lz7AFc/NJvYaGP6XWPpmpYIQHVtHTOXF/CX2ZuYs3EPcTFRXDq0Czef3oPh3dI+Ll41tXUs2lrM26sKeXdVIat2lgLQo2MS54ZG1c44pSOxJ1Aew0XlTERERBrt92+t4fWlO+iX9Z8S1i8rhe7pSSc0OtYYy7eXcP0jH9GpfTx//NxIXl+2k7/O3UJhaSXd0hO5aUwPrs3tRnojTlbILy7nnVBR+/e6XVTW1HFWv0yeuDU3bPmPl8qZiIiIRLw5G3ZzyxNzqaypwwzO7d+Jm0/vwVn9Mk/4npzlVbVMmbOZ//nHSm49owc/u+LUJk59YnRCgIiIiES8Mb078vitpzF3426uGdWN7h2TTvozE+Oi+cKZvSnYV8GfP9hIn07tuPmMnicfNkxUzkRERCSijO+b0ahLcRyveycOZEPRAX766gp6ZiRzZt/IvLNQZOx0FREREQmz6Cjj/htG0LdTO748ZQHrCvcHHemwVM5ERESkzWgXH8Njt+YSHxPFHU/PY++BqqAjfYrKmYiIiLQpOR2SeOTmUeworuCuKfOpqqkLOtInqJyJiIhImzOqRzq/uWYIH23Yw4//toxIunqFTggQERGRNumqETmsK9zPn95dT59O7fjCmb2DjgSonImIiEgb9q0L+rO+8AC/eG0lvTOTOW9AVtCRtFtTRERE2q6oKOP3nx3GoC7tuWfqQlaHbvsUaKagA4iIiIgEKSmu/gzO5PgYbn9qHrv2VwaaR+VMRERE2rwuqYn8+ZZcdu2v5EvPzKeypjawLCpnIiIiIsCwbmn87rphdEiKpaY2uLM3dUKAiIiISMilQ7tyyZAumJ3YTdabgkbORERERBoIspiBypmIiIhIRFE5ExEREYkgKmciIiIiEUTlTERERCSCqJyJiIiIRBCVMxEREZEIonImIiIiEkFUzkREREQiiMqZiIiISARRORMRERGJICpnIiIiIhFE5UxEREQkgpi7B52hyZhZEbA5zF+TAewK83fIidG2iWzaPpFN2ydyadtEtpPZPj3cPfPQia2qnDUHM8tz99ygc8inadtENm2fyKbtE7m0bSJbOLaPdmuKiIiIRBCVMxEREZEIonJ2/B4NOoAckbZNZNP2iWzaPpFL2yayNfn20TFnIiIiIhFEI2ciIiIiEUTlrJHM7CIzW21m68zs3qDztHVm9oSZFZrZsgbT0s3sLTNbG3ruEGTGtsrMupnZu2a2wsyWm9nXQtO1fSKAmSWY2VwzWxzaPj8LTe9lZnNCf+OeN7O4oLO2VWYWbWYLzezvoffaNhHCzDaZ2VIzW2RmeaFpTf63TeWsEcwsGvgTMBEYBNxgZoOCTdXmPQVcdMi0e4G33b0v8HbovTS/GuBb7j4IOB24O/T/F22fyFAJnOfuw4DhwEVmdjrwG+D/uXsfYC9wR3AR27yvASsbvNe2iSznuvvwBpfPaPK/bSpnjTMaWOfuG9y9CvgrcEXAmdo0d38f2HPI5CuAp0OvnwaubM5MUs/dd7j7gtDrUur/I5ONtk9E8Hr7Q29jQw8HzgOmhaZr+wTEzHKAS4DHQu8NbZtI1+R/21TOGicb2Nrg/bbQNIksWe6+I/R6J5AVZBgBM+sJjADmoO0TMUK7zRYBhcBbwHqg2N1rQovob1xw7gO+A9SF3ndE2yaSODDTzOab2eTQtCb/2xZzsh8gEonc3c1MpyIHyMzaAdOBr7v7vvoBgHraPsFy91pguJmlATOAAcEmEgAzuxQodPf5ZnZOwHHk8Ma7e76ZdQLeMrNVDWc21d82jZw1Tj7QrcH7nNA0iSwFZtYFIPRcGHCeNsvMYqkvZlPc/aXQZG2fCOPuxcC7wBlAmpkd/Ae7/sYFYxxwuZltov7wmfOA+9G2iRjunh96LqT+HzajCcPfNpWzxpkH9A2dMRMHXA+8EnAm+bRXgFtDr28F/hZgljYrdIzM48BKd/99g1naPhHAzDJDI2aYWSJwAfXHBb4LXBNaTNsnAO7+PXfPcfee1P935h13vxFtm4hgZslmlnLwNTABWEYY/rbpIrSNZGYXU38sQDTwhLv/IthEbZuZPQecA2QABcBPgJeBF4DuwGbgOnc/9KQBCTMzGw98ACzlP8fNfJ/64860fQJmZkOpP2g5mvp/oL/g7j83s97Uj9akAwuBm9y9MrikbVtot+Z/uful2jaRIbQdZoTexgBT3f0XZtaRJv7bpnImIiIiEkG0W1NEREQkgqiciYiIiEQQlTMRERGRCKJyJiIiIhJBVM5EREREIojKmYi0CWZWa2aLGjya7MbrZtbTzJY11eeJSNum2zeJSFtR7u7Dgw4hInIsGjkTkTbNzDaZ2f+a2VIzm2tmfULTe5rZO2a2xMzeNrPuoelZZjbDzBaHHmNDHxVtZn82s+VmNjN09X0RkeOmciYibUXiIbs1P9tgXom7DwH+SP2dQAAeAJ5296HAFOAPoel/AN5z92HASGB5aHpf4E/uPhgoBq4O668RkVZLdwgQkTbBzPa7e7vDTN8EnOfuG0I3bN/p7h3NbBfQxd2rQ9N3uHuGmRUBOQ1vn2NmPYG33L1v6P13gVh3/59m+Gki0spo5ExEBPwIr49Hw3sd1qJjekXkBKmciYjAZxs8zw69ngVcH3p9I/U3cwd4G7gLwMyizSy1uUKKSNugf9mJSFuRaGaLGrx/w90PXk6jg5ktoX7064bQtHuAJ83s20ARcFto+teAR83sDupHyO4CdoQ7vIi0HTrmTETatNAxZ7nuvivoLCIioN2aIiIiIhFFI2ciIiIiEUQjZyIiIiIRROVMREREJIKonImIiIhEEJUzERERkQiiciYiIiISQVTORERERCLI/wc5nmNVFHoWdQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(1, 1, figsize=(10, 6))\n",
    "metrics_df = pd.DataFrame({'Loss' : rnnmodel_losses, 'Epoch' : [idx for idx in range(max_epochs)]})\n",
    "axes.xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))\n",
    "sns.lineplot(data=metrics_df, y='Loss', x='Epoch', ax=axes)\n",
    "axes.set_title('Loss for epoch')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Works well :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'gangsta-nn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}