notebooks/eff_transformer/eff_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "# Based on this paper - https://arxiv.org/pdf/1706.03762.pdf\n",
    "# Might want to move layer norm inside the residual block - https://arxiv.org/pdf/2002.04745.pdf\n",
    "# Layer normalization - https://arxiv.org/pdf/1607.06450.pdf\n",
    "# TODO: Investigate learning rate warmup - https://arxiv.org/abs/2002.04745\n",
    "#!pip install torch datasets wandb"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/vik/.virtualenvs/nnets/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "import sys\n",
    "import os\n",
    "import math\n",
    "import einops\n",
    "import torch.nn.functional as F\n",
    "\n",
    "sys.path.append(os.path.abspath(\"../../data\"))\n",
    "sys.path.append(os.path.abspath(\"../../nnets\"))\n",
    "from net_utils import get_module_list\n",
    "\n",
    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "BATCH_SIZE = 32\n",
    "SP_VOCAB_SIZE = 5000\n",
    "TRAIN_SIZE = 5000"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset cnn_dailymail (/Users/vik/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/1b3c71476f6d152c31c1730e83ccb08bcf23e348233f4fcc11e182248e6bf7de)\n",
      "100%|██████████| 3/3 [00:00<00:00, 50.36it/s]\n",
      "sentencepiece_trainer.cc(177) LOG(INFO) Running command: --input=tokens.txt --model_prefix=cnn_dailymail --vocab_size=5000 --model_type=unigram\n",
      "sentencepiece_trainer.cc(77) LOG(INFO) Starts training with : \n",
      "trainer_spec {\n",
      "  input: tokens.txt\n",
      "  input_format: \n",
      "  model_prefix: cnn_dailymail\n",
      "  model_type: UNIGRAM\n",
      "  vocab_size: 5000\n",
      "  self_test_sample_size: 0\n",
      "  character_coverage: 0.9995\n",
      "  input_sentence_size: 0\n",
      "  shuffle_input_sentence: 1\n",
      "  seed_sentencepiece_size: 1000000\n",
      "  shrinking_factor: 0.75\n",
      "  max_sentence_length: 4192\n",
      "  num_threads: 16\n",
      "  num_sub_iterations: 2\n",
      "  max_sentencepiece_length: 16\n",
      "  split_by_unicode_script: 1\n",
      "  split_by_number: 1\n",
      "  split_by_whitespace: 1\n",
      "  split_digits: 0\n",
      "  treat_whitespace_as_suffix: 0\n",
      "  required_chars: \n",
      "  byte_fallback: 0\n",
      "  vocabulary_output_piece_score: 1\n",
      "  train_extremely_large_corpus: 0\n",
      "  hard_vocab_limit: 1\n",
      "  use_all_vocab: 0\n",
      "  unk_id: 0\n",
      "  bos_id: 1\n",
      "  eos_id: 2\n",
      "  pad_id: -1\n",
      "  unk_piece: <unk>\n",
      "  bos_piece: <s>\n",
      "  eos_piece: </s>\n",
      "  pad_piece: <pad>\n",
      "  unk_surface:  ⁇ \n",
      "}\n",
      "normalizer_spec {\n",
      "  name: nmt_nfkc\n",
      "  add_dummy_prefix: 1\n",
      "  remove_extra_whitespaces: 1\n",
      "  escape_whitespaces: 1\n",
      "  normalization_rule_tsv: \n",
      "}\n",
      "denormalizer_spec {}\n",
      "trainer_interface.cc(319) LOG(INFO) SentenceIterator is not specified. Using MultiFileSentenceIterator.\n",
      "trainer_interface.cc(174) LOG(INFO) Loading corpus: tokens.txt\n",
      "trainer_interface.cc(375) LOG(INFO) Loaded all 9579 sentences\n",
      "trainer_interface.cc(390) LOG(INFO) Adding meta_piece: <unk>\n",
      "trainer_interface.cc(390) LOG(INFO) Adding meta_piece: <s>\n",
      "trainer_interface.cc(390) LOG(INFO) Adding meta_piece: </s>\n",
      "trainer_interface.cc(395) LOG(INFO) Normalizing sentences...\n",
      "trainer_interface.cc(456) LOG(INFO) all chars count=479969\n",
      "trainer_interface.cc(467) LOG(INFO) Done: 99.9556% characters are covered.\n",
      "trainer_interface.cc(477) LOG(INFO) Alphabet size=69\n",
      "trainer_interface.cc(478) LOG(INFO) Final character coverage=0.999556\n",
      "trainer_interface.cc(510) LOG(INFO) Done! preprocessed 9579 sentences.\n",
      "unigram_model_trainer.cc(138) LOG(INFO) Making suffix array...\n",
      "unigram_model_trainer.cc(142) LOG(INFO) Extracting frequent sub strings...\n",
      "unigram_model_trainer.cc(193) LOG(INFO) Initialized 29939 seed sentencepieces\n",
      "trainer_interface.cc(516) LOG(INFO) Tokenizing input sentences with whitespace: 9579\n",
      "trainer_interface.cc(526) LOG(INFO) Done! 17665\n",
      "unigram_model_trainer.cc(488) LOG(INFO) Using 17665 sentences for EM training\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=0 size=12519 obj=12.7854 num_tokens=36663 num_tokens/piece=2.92859\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=1 size=10779 obj=10.9544 num_tokens=37064 num_tokens/piece=3.43854\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=0 size=8081 obj=10.9733 num_tokens=39373 num_tokens/piece=4.87229\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=1 size=8075 obj=10.9012 num_tokens=39418 num_tokens/piece=4.88149\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=0 size=6054 obj=11.1673 num_tokens=43264 num_tokens/piece=7.14635\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=1 size=6054 obj=11.0829 num_tokens=43273 num_tokens/piece=7.14784\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=0 size=5500 obj=11.1838 num_tokens=44589 num_tokens/piece=8.10709\n",
      "unigram_model_trainer.cc(504) LOG(INFO) EM sub_iter=1 size=5500 obj=11.1563 num_tokens=44602 num_tokens/piece=8.10945\n",
      "trainer_interface.cc(604) LOG(INFO) Saving model: cnn_dailymail.model\n",
      "trainer_interface.cc(615) LOG(INFO) Saving vocabs: cnn_dailymail.vocab\n"
     ]
    }
   ],
   "source": [
    "# TODO: Fix dataset to work with new format\n",
    "# TODO: Improve dataset to optimize padding and other things (drop incomplete, etc)\n",
    "from text_data import WikiTextDataset\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_wrapper = WikiTextDataset(tokenizer_vocab=SP_VOCAB_SIZE, download_split_pct=\"5%\")\n",
    "train_dataset = train_wrapper.process_dataset()\n",
    "train_dataset = train_dataset.train_test_split(test_size=0.1)\n",
    "\n",
    "train_split = train_dataset[\"train\"]\n",
    "train = DataLoader(train_split, batch_size=BATCH_SIZE)\n",
    "\n",
    "valid_split = train_dataset[\"test\"]\n",
    "valid = DataLoader(valid_split, batch_size=BATCH_SIZE)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# Add in ROPE embedding\n",
    "class ROPE(nn.Module):\n",
    "    def __init__(self, embedding_dim, seq_len):\n",
    "        super(ROPE, self).__init__()\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.seq_len = seq_len\n",
    "\n",
    "        self.cos_embeds = torch.zeros(seq_len, embedding_dim, device=DEVICE)\n",
    "        self.sin_embeds = torch.zeros(seq_len, embedding_dim, device=DEVICE)\n",
    "\n",
    "        embed_pos = 10000 ** (-2 * torch.ceil((torch.arange(0, embedding_dim) + 1) / 2) / embedding_dim)\n",
    "        for i in range(0, seq_len):\n",
    "            self.cos_embeds[i] = torch.cos((i + 1) * embed_pos)\n",
    "            self.sin_embeds[i] = torch.sin((i + 1) * embed_pos)\n",
    "\n",
    "        self.indices = torch.zeros(self.embedding_dim, device=DEVICE, dtype=torch.long)\n",
    "        self.mask = torch.zeros(self.embedding_dim, device=DEVICE, dtype=torch.int)\n",
    "        for i in range(0, embedding_dim, 2):\n",
    "            self.indices[i] = i + 1\n",
    "            self.indices[i+1] = i\n",
    "\n",
    "            self.mask[i] = -1\n",
    "            self.mask[i+1] = 1\n",
    "\n",
    "\n",
    "    def rotate(self, x):\n",
    "        return x[...,self.indices] * self.mask\n",
    "\n",
    "    def forward(self, x):\n",
    "        current_val = x * self.cos_embeds[:x.shape[-2],:]\n",
    "        next_val = self.rotate(x) * self.sin_embeds[:x.shape[-2],:]\n",
    "        return current_val + next_val\n",
    "\n",
    "\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, input_units, attention_heads, mask=False):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.input_units = input_units\n",
    "        self.attention_heads = attention_heads\n",
    "        self.head_units = int(input_units/attention_heads)\n",
    "        self.mask = mask\n",
    "\n",
    "        k = math.sqrt(1/self.input_units)\n",
    "        # Drop bias\n",
    "        # Single kv head\n",
    "        self.kv_proj_weight = nn.Parameter(torch.rand(2, input_units, self.head_units) * 2 * k - k)\n",
    "        self.q_proj_weight = nn.Parameter(torch.rand(input_units, self.attention_heads * self.head_units) * 2 * k - k)\n",
    "        self.out_proj_weight = nn.Parameter(torch.rand(self.attention_heads * self.head_units, input_units) * 2 * k - k)\n",
    "\n",
    "        # 1024 is max sequence length\n",
    "        self.rope = ROPE(self.head_units, 1024)\n",
    "\n",
    "    def forward(self, queries, keys, values):\n",
    "        # convert to 4d tensor with batch_size, attn_heads, seq_len, embedding_dim\n",
    "        proj_queries = torch.einsum(\"...se, eo->...so\", queries, self.q_proj_weight)\n",
    "        proj_queries = proj_queries.view(queries.shape[0], queries.shape[1], self.attention_heads, self.head_units).swapaxes(1,2)\n",
    "        proj_queries = self.rope(proj_queries)\n",
    "\n",
    "        proj_keys = torch.einsum(\"...se, eo->...so\", keys, self.kv_proj_weight[0])\n",
    "        proj_keys = proj_keys.view(keys.shape[0], keys.shape[1], self.head_units)\n",
    "        proj_keys = self.rope(proj_keys)\n",
    "\n",
    "        proj_values = torch.einsum(\"...se, eo->...so\", values, self.kv_proj_weight[0])\n",
    "        proj_values = proj_values.view(values.shape[0], values.shape[1], self.head_units)\n",
    "\n",
    "        attention = torch.einsum(\"baqh, bhk->baqk\", proj_queries, torch.transpose(proj_keys, -1, -2)) / np.sqrt(proj_keys.shape[-1])\n",
    "        if self.mask:\n",
    "            # Prevent decoder queries from looking at tokens that come after\n",
    "            # Do this by setting attention to negative infinity, so it is softmaxed to zero in the next step\n",
    "            mask = torch.full((attention.shape[-2], attention.shape[-1]), -torch.inf, device=DEVICE)\n",
    "            attention += torch.triu(mask, diagonal=1)\n",
    "\n",
    "        # Softmax on last dimension\n",
    "        # Sequence-wise softmax, so attention between one sequence and other sequences sums to 1\n",
    "        attention = torch.softmax(attention, dim=-1)\n",
    "        weighted_values = torch.einsum(\"baqk, bke->baqe\", attention, proj_values)\n",
    "\n",
    "        # Swap attention head and sequence axis, then reshape to batch, seq, embedding\n",
    "        weighted_values = weighted_values.swapaxes(1,2).reshape(queries.shape[0], queries.shape[1], -1)\n",
    "        weighted_values = torch.einsum(\"...se, eo->...so\", weighted_values, self.out_proj_weight)\n",
    "        return weighted_values"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "class SwiGLU(nn.Module):\n",
    "    def __init__(self, input_units, hidden_units):\n",
    "        super(SwiGLU, self).__init__()\n",
    "        self.linear1 = nn.Linear(input_units, hidden_units, bias=False)\n",
    "        self.linear2 = nn.Linear(input_units, hidden_units, bias=False)\n",
    "        self.linear3 = nn.Linear(hidden_units, input_units, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x1 = self.linear1(x)\n",
    "        swish = self.sigmoid(x1) * x1\n",
    "        x2 = self.linear2(x)\n",
    "        swiglu = self.linear3(swish * x2)\n",
    "        return swiglu\n",
    "\n",
    "class DecoderBlock(nn.Module):\n",
    "    def __init__(self, input_units, attention_heads, hidden_units=2048, dropout_p=.1):\n",
    "        super(DecoderBlock, self).__init__()\n",
    "        self.in_attn = MultiHeadAttention(input_units, attention_heads, mask=True)\n",
    "        self.dropouts = get_module_list(2, nn.Dropout, dropout_p)\n",
    "        # Drop bias\n",
    "        self.lns = get_module_list(2, nn.LayerNorm, input_units)\n",
    "        # Switch to swiglu from two linear layers\n",
    "        self.swiglu = SwiGLU(input_units, hidden_units)\n",
    "\n",
    "    def forward(self, x):\n",
    "        weighted_values = self.dropouts[0](self.in_attn(x, x, x))\n",
    "        # Pre normalization\n",
    "        x = x + self.lns[0](weighted_values)\n",
    "\n",
    "        reprojected = self.dropouts[1](self.swiglu(x))\n",
    "        # Pre normalization\n",
    "        x = x + self.lns[1](reprojected)\n",
    "        return x"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "class Transformer(nn.Module):\n",
    "    def __init__(self, input_units, hidden_units, attention_heads, max_len=256, blocks=1):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.blocks = blocks\n",
    "        self.dropouts = get_module_list(2, nn.Dropout, .1)\n",
    "        self.decoders = get_module_list(blocks, DecoderBlock, hidden_units, attention_heads)\n",
    "\n",
    "        self.embedding = nn.Parameter(torch.empty(input_units, hidden_units))\n",
    "        nn.init.xavier_uniform_(self.embedding)\n",
    "        self.input_units = input_units\n",
    "\n",
    "    # Tie input output weights\n",
    "    def embed(self, x, reverse=False):\n",
    "        if reverse:\n",
    "            return x @ self.embedding.T\n",
    "        else:\n",
    "            embedded = self.embedding[x.to(torch.long).view(-1)]\n",
    "            return embedded.view(x.shape[0], x.shape[1], -1)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        dec_outputs = self.dropouts[1](self.embed(x))\n",
    "        for i in range(self.blocks):\n",
    "            dec_outputs = self.decoders[i](dec_outputs)\n",
    "\n",
    "        token_vectors = self.embed(dec_outputs, reverse=True)\n",
    "        return token_vectors"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "def generate(sequence, pred, target, wrapper):\n",
    "    prompts = wrapper.decode_batch(sequence[:,:wrapper.x_length].cpu())\n",
    "    texts = wrapper.decode_batch(torch.argmax(pred[:,wrapper.x_length:], dim=2).cpu())\n",
    "    correct_texts = wrapper.decode_batch(target[:,wrapper.x_length:].cpu())\n",
    "\n",
    "    displays = []\n",
    "    for p, t, ct in zip(prompts, texts, correct_texts):\n",
    "        displays.append(f\"{p} | {ct} | {t}\")\n",
    "    return displays"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001B[34m\u001B[1mwandb\u001B[0m: Currently logged in as: \u001B[33mvikp\u001B[0m. Use \u001B[1m`wandb login --relogin`\u001B[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "Tracking run with wandb version 0.13.9"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "Run data is saved locally in <code>/Users/vik/Personal/nnets/notebooks/eff_transformer/wandb/run-20230129_215805-2ymycngo</code>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": "Syncing run <strong><a href=\"https://wandb.ai/vikp/decoder-only/runs/2ymycngo\" target=\"_blank\">split-embed</a></strong> to <a href=\"https://wandb.ai/vikp/decoder-only\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": " View project at <a href=\"https://wandb.ai/vikp/decoder-only\" target=\"_blank\">https://wandb.ai/vikp/decoder-only</a>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "<IPython.core.display.HTML object>",
      "text/html": " View run at <a href=\"https://wandb.ai/vikp/decoder-only/runs/2ymycngo\" target=\"_blank\">https://wandb.ai/vikp/decoder-only/runs/2ymycngo</a>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "[]"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "import wandb\n",
    "import time\n",
    "\n",
    "wandb.init(project=\"decoder-only\", notes=\"Split embedding weights\", name=\"split-embed\")\n",
    "\n",
    "# TODO: Profile and improve perf - https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html\n",
    "model = Transformer(wrapper.vocab_size, 512, 8, blocks=6).to(DEVICE)\n",
    "loss_fn = nn.CrossEntropyLoss(ignore_index=wrapper.pad_token)\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)\n",
    "wandb.watch(model, log_freq=100)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 train loss: 0.22401335331900366\n",
      "Javon Thompson is believed to have died in December 2006 | . One Mind Ministries members prayed for his re | .ssedssssss tos . .\n",
      "Mom discovers young boy's guinea pig has died su | ddenly . She decides to keep it a secret because he | . . . .,sedsssssss to\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.25s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.20363730937242508\n",
      "Smartwatches have hidden, darker side the companies selling | them are unlikely to talk about, says Andre Spicer . | .sssssssssssss\n",
      "Bafetimbi Gomis collapses within 10 minutes of | kickoff at Tottenham . But he reportedly left the  | sss .sssssssssss\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 train loss: 0.1914054452345289\n",
      "Residents in Huntington, Virginia, voluntari | ly evacuating because of flooding . FEMA says no states | . .,ssssss The \" \"sss\n",
      "96 Liverpool fans died as a result of the Hillsborough disaster on | April 15, 1989 . Fans were crushed against sta | . . . . .s Ass as a a a a\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2 train loss: 0.16946742514084126\n",
      "Woman travels from Kenya to India for a tricky third kidn | ey transplant . Lorna Irungu suffers from l | ss C::ssalesingmal\n",
      "British graphic artist's identity remains a mystery desp | ite huge popularity . Feted by the art world and Hollywood | essss . . The.s the the s the \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3 train loss: 0.14704467232885032\n",
      "Seminole Tribe owns Hard Rock properties, look | s to help unify Native Americans . The Native American Group | s of Mo coach coach coach mu . . . The Thes  are\n",
      "Site provided names, IP and e-mail addresses | of offenders it found, AG says . Similar inform | ing youthing . . . . . says . Twoinginging Ma\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4 train loss: 0.12437003239475447\n",
      "NEW: Friends, relatives of three missing men to organize private search | . Coast Guard suspends search off Florida coast at sundown Tuesday . | . NEW Guard rescueds rescued, rescuedton search Floridas Florida . .\n",
      "The Beatles' remastered catalogue will be released in September . | Release includes all 12 Beatles albums and two later works . Beatles | rotess,,sss, Beatles,,,, Beatles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:44,  3.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5 train loss: 0.10707830098168604\n",
      "Boy, 12, found .25-caliber gun in box in | closet . Police have not charged anyone in relation to incident . | closet . closet charge charge charged toone relat Nataed to incident .\n",
      "Britain is ready to increase troop numbers in Afghanistan, arm | y chief says . Richard Dannatt told the Times up | y chiefy . Richard Richardish Dan Danish a a at a\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:44,  3.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6 train loss: 0.09431973294965153\n",
      "Obama sends nuclear agreement with the United Arab Emirates to the | Senate . Recently released video shows member of UAE royal | lyelylylylylylys tos U.EE\n",
      "Russian gas giant Gazprom says it will cut gas supply | 10 a.m. Thursday  | 10 10 10....m...m...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:44,  3.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7 train loss: 0.0822273473801284\n",
      "Congressman from Rhode Island seeks help against substance abuse . | Rep. Patrick Kennedy, 41, acknowledges long struggle with addiction | Rep: acknowledgek acknowledge Kennedy acknowledge, acknowledges need,d addiction\n",
      "Kellogg's donated two tons of cer | eal to the San Francisco Food Bank . Company dumped the c | alalal San San Sanco Food Bank . Company dumped the c\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8 train loss: 0.07359302128697263\n",
      "Paul McCartney, Ringo Starr say their music sound | s better . The former Beatles talk to CNN's Larry | story better . The 70former Photo talk of be's Larry\n",
      "Russian gas giant Gazprom says it will cut gas supply | 10 a.m. Thursday  | a a a a. a am ammm.mm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9 train loss: 0.06514370634124197\n",
      "H.W. Brands: Roosevelt-Washington | White House dinner was controversial . He says the the opposition was about | dinner dinner dinner . dinner . controvers He says controvers controvers controverspposition was seek\n",
      "Mark Penn is CEO of public relations giant Burson-Marsteller | . Penn met with Colombia ambassador over trade pact . Colombia fire | . Colombia met a Colombia ofmbassador a its Colombiaador of Colombian\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10 train loss: 0.060922711291189854\n",
      "Barcelona are ready to let Brazilian international Ronaldinho leave the club | . Club president Joan Laporta says that Ronaldinho needs | . Hevent 1 saysa saystoers says that Ronalda thats\n",
      "Essence magazine interviews senator, wife and children in Chicago | home . Editor says candidate, Michelle Obama try to keep daughter | follow . Cofftor says Michel, Michel,le should to keeple\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.23s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.3429071456193924\n",
      "Jina Krause-Vilmar: On International Women' | s Day, focus on helping refugee women adapt to new | day . North economy, 27, CESis, Council and\n",
      "Parents of teens should have a criminal defense lawyer in mind  | -- just in case, says author . Author Lisa Green calls this the | . Pilots, 90ron, but was the the airc,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11 train loss: 0.050278863978796995\n",
      "At least 98 dead, many more missing, from flash | floods in Jakarta . Flood began after heavy rain | ining in 11karta . Froundus imping after rain rain\n",
      "Pakistani President insists his state's nuclear arsenal | is safe, despite Taliban gains . Recent gains by the Taliban | deal safe deal CNNite desp .s . Recent dealed whe the U\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12 train loss: 0.04485224521365659\n",
      "NEW: Sen. Obama says any recover plan must help workers and home | owners . McCain repeats misleading charge against Obama on taxes | owners . McCain repeats mist iting mordst it on taxes\n",
      "International Criminal Court: Sudan \"taken no steps\" to | arrest Darfur war criminals . One suspect now in charge of humanitarian | charge shooting suspect .s . One suspect now charge charge of chargearian\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13 train loss: 0.0406720953640239\n",
      "Troops to stay until 2011, with the stipulation that NATO | contribute more forces . Most of Canada's 2,500 troops | se more forces . weatherst of Canada's a,500 troops\n",
      "\"Several heavy metals\" found in levels abo | ve safe drinking-water standards . TVA pledges cleanup; | ve drinking drinkingwaterwater standard . . TV . pledges cleanup;\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14 train loss: 0.03537089057780545\n",
      "Iran set to send female athlete to next year's Winter Olympics for | the first time . One female skier, three male skiers will be | Darfur time time . One, skier, \" male \"ers, be\n",
      "Ex-PM Thaksin Shinawatra convicted of corruption, sentenced | to jail . Court rules Thaksin facilitated wife's purchase | . jail . Court rules servein facilitdated wife ins purchase\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15 train loss: 0.03182108311046814\n",
      "NEW: California Public Utilities Commission passed ban Thursday . | Phone records show engineer texting during work hours on | Phone records shows texts onring t ons on\n",
      "The Rev. Jeremiah Wright seeks to explain the | ology of black church . Wright says criticisms come from those | ology of black church . Wright says criticisms know from hose\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16 train loss: 0.02937748265163652\n",
      " | new Terminal 5 . The new building took more than 15 | er Terminal 5 . The building building took more than 15\n",
      "NEW: Gunfire from pirates forces sailors, who did not return | fire, to turn back . NEW: FBI launches criminal investigation into hijack | to, to turn back . NEW: FBI launches criminal investigation into hijack\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17 train loss: 0.025108223151544044\n",
      "NEW: Blagojevich declines comment, says \"I'm enjoying Dis | ney World with my kids\" Blagojevich faces up to 20 years in prison | s World my my heart\" Bara faces World to 20 prison to prison\n",
      "NEW: All vehicles banned in city of Amara through Thursday, | provincial governor says . Death toll continues to rise as In | provincial governor says by Death toll continues toll for In\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18 train loss: 0.02279952210855895\n",
      "David Cook became the seventh \"American Idol\" in the spring . | Cook wrote or co-wrote most songs on debut, even | Cook wrote or co-wroterote songs, debut on debut\n",
      "Celebrated Nigerian author has been resident in New York for over 20 years | . Most famous work, \"Things Fall Apart\" tra | countr Most famous A, \"This Fall Fall Fallpart Fall out\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19 train loss: 0.028387771970753012\n",
      "Fighting comes day after Rwanda arrests Tutsi rebel leader, Lauren | t Nkunda . Neighboring nations have been on different | t .k N . Neighboring mans have been Shi different\n",
      "Maryland couple operates Rude Ranch Animal Rescue out | of their home . Bob and Katherine Rude currently care | of their home . People and Kathdeine keepde inly in\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20 train loss: 0.022372872343864935\n",
      "1 person dead, 116 ill in northeastern part of the state . | Focus of investigation is restaurant in Locust Gro | Fcc0pa investigation under under about Louump Gro\n",
      "Heavyweight champ Jack Johnson was arrested around 1908 . His offense: | Transporting a white woman across state lines for \"immoral | Transporting a white woman a line other lines for 40immoral\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.17s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.43674537539482117\n",
      "Actor Ashton Kutcher complained on Facebook that men' | s rooms don't have diapering tables . | ra hundreds out of the theft as and why inflation .\n",
      "The abducted workers were seized in an attack on an oil field last | week . They are from the Philippines, Bangladesh, Austria, the | \"gs \"gight\" Auduned to goet,\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21 train loss: 0.01858798562838086\n",
      "Ex-presidents of Mexico, Brazil and Colombia urge decriminal | ization of marijuana . Leaders urge treatment for addicts, | ization would marijuana . Leaders should treatment for addiction,\n",
      "Organ shortage in rich states has created a trade from poorer | countries . \"Transplant tourists\" travel to poor countr | countries . \"Transplant tourists\" travel to poor countr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22 train loss: 0.01736916198298849\n",
      "Passenger questioned about large amount of cash . He says the money was | from sale of political merchandise . ACLU | from sale of political pmerleandisisis ACLU\n",
      "Swat Valley region used to be a popular destination for tourists and skiers . | Taliban are imposing their strict brand of Islamic law in | Taliban are imposing their strict brand of Islamicc in\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23 train loss: 0.01890337569317941\n",
      "Brittanee Drexel disappears on trip to Myrtle Beach, South Carolina | . Mom says she spoke to daughter but didn't know she was | . Mom says she spoke to daughter but didn't know she was\n",
      "Nicole Suveges, 38, was part of team of a | cademic embeds advising military in Iraq . The political scientist | cademic embss advis military by killed Iraq . The political leader\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24 train loss: 0.018238013468939683\n",
      "U.S. calls remarks vile, hateful, inc | iteful, praises U.N. condemnation . Dozens go | iteful, praises U.N. praiseation says Dozens go\n",
      "Former FBI agent Bob Levinson disappeared in Iran in March 2007 . His | wife says she has done everything to draw attention to Levinson's | wife says she has a everything to draw attention to Levinson Jos\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25 train loss: 0.016980656300639285\n",
      "Large Hadron Collider will have first attempt at circulating a | beam September 10 . It's the largest particle ac | beam September 10 . It's large largest bearticle ac\n",
      "Randomly chosen, undecided Democrats will | watch debate while turning dials . Dials will rate | watch debate Howhile turning dial . . Dial poor will rate\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26 train loss: 0.01775279429195256\n",
      "Cirque Du Soleil's \"Kooza\" | went looking for juggler with 11 world records | 1,went looker for juggler with Friday world records\n",
      "Tamil rebels say Sri Lankan forces ignoring orders to | end military operations Sri Lanka: Forces ordered to cease | end military operations:ri Lanka: Forceseded message cease\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27 train loss: 0.012975201640149642\n",
      "Celtic defeat rivals Rangers 2-0 to win Scottish League | Cup final at Hampden . Extra-time goals by | Cup final at Hampshir . Extra-time goals a\n",
      "New digital topographic map reveals more of Earth than | ever before . Images were taken by Japanese imaging | ever before . Images were taken by Japanese imaging\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28 train loss: 0.010615310782630895\n",
      "Pakistani Taliban's interpretation of sharia includes banning | girls from school . Deal with the Taliban comes after a visit | girl school schoolm school . Deal with \" Taliban,s fro a visit\n",
      "48-year-old Boy George on trial in London in November . Per | former faces charges of false imprisonment relating to 2007 incident . Georg | former faces imprison of false imprisonment .ing to 2007 incident . Georg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29 train loss: 0.009465205194107417\n",
      "Most common car safety system is ABS or anti-lock | brake system . EBD: subsystem of | brake system . EB:: subure systeme system of\n",
      "More than 600 supporters watch their team suffer its sixth los | s in seven games . Energie Cottbus are second | s in seven games . Energie Cottbusive second\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30 train loss: 0.009462838530026633\n",
      "Liu Yan is paralyzed while rehearsing her | solo dance for the Olympics' opening . Doctors say she will not | solo dance for the Olympics'sing . Doctors say she had be\n",
      "Second-placed Ajax held 2-2 at home by V | itesse Arnhem in Dutch Eredivisie . | itesee Arnmm Dutch Dutch Eredivisie .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.23s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.5029298365116119\n",
      "Red Bull's No.1 driver Daniel Ricciardo says Form | ula One is a \"crueler sport\" Last year | ation . Man was amost drop to those shot Monday, spokesman,\n",
      "Jina Krause-Vilmar: On International Women' | s Day, focus on helping refugee women adapt to new | on curtenies, Chond . In0 have presented the\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31 train loss: 0.012501134479354168\n",
      "NEW: Paper says rejection part of standard back-and-forth | procedure . John McCain writes essay in defending his Iraq policy | procedure . John McCain's essay  defending his Iraq policy\n",
      "Suit alleges crew sent into pirate-infested waters without protection . | Representatives of ship's owners have yet to comment . | Representatives of ship's owners have yet to comment .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32 train loss: 0.011527195906844633\n",
      "Ruben Navarrette: McCain tried to get under Obama's skin with attack | s . Obama remained cool and wound up ending strongly in the | s . Obama remained for thing \"ed ended strongs in\n",
      "Lawyer who blamed Guatemala's president for two slayings was killed Sunday | . In video released after his death, lawyer says to blame president if | . In video,, his wife, lawyers he blame president if\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33 train loss: 0.01121688674621541\n",
      "Pregnant woman reportedly found husband, other man dead in a | partment . Slain students were international Ph.D. candidate | partment . Slain .s were international Ph.D. candidate\n",
      "Trump advises to take advantage of low prices, get seller | to do the financing . Trump: Obama rebuilding U. | to do the fillivo . Trump: Obama rebuilding U.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34 train loss: 0.009480949645412379\n",
      "Poland was ruled by Soviet-backed regime after the Second World War | . Solidarity movement became a key factor in the fall of | . Solidarity movement became a key factor in the fall of\n",
      "Adam Sandler wore cast for part of \"Bedtime S | tories\" filming . Some scenes required a body double after star | tories\" filming . Some scenes required a body double after star\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35 train loss: 0.012617782914432985\n",
      "A good corn-gluten mix cures brown patch | es and weakens most fungi . Brown patches are often | es and weakens most fungi . Brown patches are often\n",
      "\"Slumdog Millionaire\" wins eight Oscars, including best picture | and director . Sean Penn wins best actor for \"Milk\" Kat | and director . Sean Penn wins Rock actors \"Myk\" Kat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36 train loss: 0.013486170826544021\n",
      "Thousands of items sent from around globe are housed on Tech | campus . Items range from letters from kids | campus . Items range from letters from kids\n",
      "NEW: Suspected spree killer described as meth addict | . Nicholas T. Sheley, 28, did not enter a | . Nichola Green T. Sheley due did with did not enter a\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37 train loss: 0.012369921341024596\n",
      "Fire reported as plane goes off runway in freezing rain at Lub | bock, Texas, airport . Two crew members hospitalized with apparent | bock, Texas, Texas . Two crew members hospitalized with apparent\n",
      "Rodent the size of small car discovered in Urugua | y . Scientists say the rodent must have weighed 1,000 | y . Scientists say the extagent must have weighed 1,000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38 train loss: 0.008941682017055051\n",
      "NEW: Workers release one of five people being held hostage at Ca | terpillar factory . Workers angry that Caterpillar | terpillter factory . Workers angry that Caterpillra\n",
      "Authorities order evacuation of 8,000 people who live near Galeras volcano | . No injuries reported; volcano erupts for second time in less than a | . No injuries says; volcano erupts for second time in less than battle\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39 train loss: 0.01142408332048819\n",
      "NEW: First United Nations group to address piracy meets in New York | . Two or three speedboats were chasing the Dutch contain | . Two or three speedboats were chasing the air contain\n",
      "Director Antoine Fuqua is filmmaker behind CNN's \"From | MLK to Today\" Fuqua discusses life, experiences with | breaLK to Today\" Fuqua discusses life, experiences with\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40 train loss: 0.009240936725560962\n",
      "Tom Wilkerson: Pirates based in Somalia represent threat to crew | s and cargo . He says U.S. strategy so far has | s and cargo . He says U.S. far so far has\n",
      "French FM Kouchner has told France to prepare for po | ssibility of war with Iran . Was a surprise appointment | ssualbility of war with Iran . Wasual surprisedment\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.27s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.5354069173336029\n",
      "Pope has talked of retirement before, but this time he says he | think papacy will end after no more than five years . Francis | 's family . Gen. Citepew about self,\n",
      "Two American women arrested for carving initials into a Coloss | eum wall . Meanwhile, Egypt investigating Russian porn | dam Hand of an end . Oneic land in an au\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41 train loss: 0.008090863280512136\n",
      "New Orleans elbowed New York aside as the | best city for fine dining . Portland, Oregon, swe | best city for fine dining . Portland, Oregon, swe\n",
      "London football club Chelsea appoint Carlo Ancelotti as their | new manager . Ancelotti has been in charge of Italian giant | new manager . Ancels has been in charge of Italian giant\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42 train loss: 0.00644427841240219\n",
      "Isaac Toussie was involved in mortgage scheme | in New York . Answers wanted about how thorough of an | in New York . Answers wanted about how thorough of an\n",
      "Kabul hospital's operating budget is less than $1,200 a month . | Government pays salaries, but even basic supplies | Government pays salarar, but evenicbasic supplies\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43 train loss: 0.005888602222669227\n",
      "Jessica Cox, 25, was born with no arms . Accusto | med to not having arms, she decided against using | med to not having arms, she decided against noting\n",
      "Mexican media says local chief put under house arrest for 45 days . With Francis | co Velasco Delgado's removal, | co Velasco engagegad M's removal,\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44 train loss: 0.007377512388509409\n",
      "Women older than 55 make up the fastest-growing age group | on Facebook . Expert says the site has hit a \"tipp | on Facebook . Expert sai the site has hit a \"Wive\n",
      "Presley earned more than Justin Timberlake  | 44M | 44M,, Madonna proud$40M last last year . Cartoon\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45 train loss: 0.010811761856592935\n",
      "NEW: The woman became ill on January 5 and died on Saturday . | NEW: Two tests on the woman are positive for H5N1  | NEW: Two tests on the woman are positive for H5N1 .\n",
      "Attorney general: Deputy shot himself twice in the chin, once | in side of the head . Forensic examination on de | in side of the head . Forensic examin' on de\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46 train loss: 0.007335285280413668\n",
      "Gulfport, Mississippi, mayor \"will continue working to rebuild  | our city\" Mayor Gregory Brent Warr, says he pleaded | our city\" Mayor Gregory Brent Warr says says he pleaded\n",
      "Chris Nowinski played football at Harvard, then wrestle | d professionally . Still suffers effects of six concus | d professionally . Still suffers effects of six conacceptus\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47 train loss: 0.0054973444869292195\n",
      "Former FBI agent Bob Levinson disappeared in Iran in March 2007 . His | wife says she has done everything to draw attention to Levinson's | wife says she has done everything to draw attention to Levinson's\n",
      "Palin meets with world leaders on sidelines of U.N. world | summit . Meeting with Kissinger stretches into an hour | summit . Meeting with Kisseer intoes into an hour\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48 train loss: 0.009034820230966755\n",
      "NEW: Two Canadians, yachting tycoon, sister of | Bollywood actor among dead . Rabbi Gavri | Bolly, actor among dead in Rabbb Glyri\n",
      "NEW: Barack Obama, John McCain lay roses at ground zer | o . New polls show both candidates with slim lead | o . New polls show both candidates in s inm lead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49 train loss: 0.012979863185820908\n",
      "A McDonald's in Washington goes silent as patro | ns watch inaugural . \"This is America happening,\" | ns watch inaugural . \"This is America happening\"\"\n",
      "Shaun Gopaul, Alex Jimenez served together in military | starting in 2005 . Gopaul: \"He saved our | starting the 2005 . 48-pa says: \"He saved theour\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50 train loss: 0.021682912869186236\n",
      "Opposition parties seek to oust Prime Minister Stephen Harper's | government . Harper's Tories gained seats in Canada's | government . Harper's gainies gained seats in Canada's\n",
      "Expert: Trial of Nazi war crimes suspect John Demjanjuk could be last of its | kind . Leading Nazis prosecuted at Nur | kind . Leading delays pros shouldcuted at Nur\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.23s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.5322788059711456\n",
      "A Civil War ironclad must be moved so the shipp | ing channel can be deepened . Wreckage lies a couple miles | . The practices that Manchester tied Trafford club for Argentin\n",
      "Bafetimbi Gomis collapses within 10 minutes of | kickoff at Tottenham . But he reportedly left the  | y in Merrill Lynch politics . Worked by Japan to 27 coins\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:59,  2.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51 train loss: 0.014196024886493025\n",
      "Spain commemorates fifth anniversary of Madr | id train bombings . March 11, 2004, attacks killed 191 | demic train bombings . March 11, 2004, attack, 19 191\n",
      "Bollywood superstar makes sporting debut as owner of Kolk | ata Knight Riders . Shah Rukh Khan still wildly | atu Knight wildiders . Shah Ruk wild Khan still wildly\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:31,  6.84s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52 train loss: 0.006850670865769016\n",
      "Rafael Nadal defeats David Ferrer in stra | ight sets to lift Barcelona Open title . World number one wins 6-2 | ight sets to lift Barcelona Open title . World number one of 6-2\n",
      "Prince Harry describes Princess Diana as \"the best mother in the | world\" He asks for her to be remembered as \"fun- | world\" He asks for her to be remembered out \"fun-\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [15:46,  6.53s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53 train loss: 0.004562510073120738\n",
      "NEW: Opposition accuses PM Harper of putting his job ahead of Canada | 's interests . Move postpones opposition parties' | 's interests . Move postpones opposition parties'\n",
      "Survey: 68 percent of U.S. hotels said they had energy | -efficient lights . InterContinental aims to cut energy | -efficient lights . InterContinental aims to cut energy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 54 train loss: 0.003947840365260069\n",
      "White House says Obama is not going to make any decision that imper | ils safety . Senate votes 90-6 on a measure to | ils safety . Justicee votes 90-6 only measures\n",
      "Analyst: Corporate lobbyists will not be only ones | heard by Obama's transition staff . Transition team' | heard by Obama's transition staff . Transition team'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:10,  7.52s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55 train loss: 0.004128364022371584\n",
      "CNN's MainSail host Shirley Robertson is a double Olympic gold | medalist . Robertson has never climbed a big mast before | medalist . Indianson has never climbed a big mast before\n",
      "Hillary Clinton owes $2.3 million in campaign debt from | failed 2008 presidential bid . Her campaign committee reported having $2.6 | failed 2008 presidential bid . Her campaign committee reported having $2.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:44,  6.93s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56 train loss: 0.004081584033074564\n",
      "Nations hoping for deal in which Libya would compensate terrorism victims . | Eights acts would be covered in possible agreement . Libya has | Eights acts would be covered in possible agreement . Libya has\n",
      "Debra Lafave's probation forbids her | to have contact with anyone under 18 . Florida's Corrections Department | to have contact under anyone under 18 . Florida's Corrections Department\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:44,  3.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57 train loss: 0.004743582007057708\n",
      "White House spokeswoman Dana Perino gave last news briefing Friday . | Perino, 36, succeeded Tony Snow, is only second | Perino, 36, succeeded Tony Snow, is only second\n",
      "Government may use eminent domain to seize land needed for | 9 | 91111 memorial . Hijacked United Flight 93 crashed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [14:50,  6.14s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58 train loss: 0.006751429034268548\n",
      "Tino Schaedler, colleagues talk about NAU, | a remote collaboration project . They discuss the concept of | TV remote collaboration project . They discuss the concept of\n",
      "Some 200 Palestinian refugees from Iraq will go to Iceland, Sweden . | 2,300 Palestinians \"are living in desperate | 2,300ars \"are living in d this\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:55,  2.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59 train loss: 0.006731619769386177\n",
      "Harry serves in British Army and spent 10 weeks in Afghanistan this | year . Decision was made to pull prince from Afghanistan a | year . Decision made made to pullcknit from Afghanistan a\n",
      "Indian army troops use helicopters, boats to deliver supplies, rescue village | rs . NEW: Survivor says he lost wife, children, parents, nin | rs . NEW: Survivor says he lost wife, children, children, nin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 60 train loss: 0.0060081466004766266\n",
      "China extends deadline for Internet filtering software to accompan | y PCs . International backlash, unreadiness of | s PCs . International backlash, unreadiness of\n",
      "Old dance hall one of few underground gay clubs in Sha | nghai for older generation . Caters for a differen | nghai for older physicalration . Caters for differen differen\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.21s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.5874077379703522\n",
      "Media coverage of ISIS could spur \"real | wave of Islamophobia,\" report's edit | , for a grown from tran's debate . Mayor Lev\n",
      "Interim report exposes delays and inaction after MH3 | 70 disappeared . Flight carrying 239 people and crew has | medale on six . Cupuritbur scored by\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [03:43,  1.54s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61 train loss: 0.006889753351951467\n",
      "Government gives First Secretary Mark Sullivan 48 hours to | leave the country . Sullivan is accused of meddling in | rican the country . Sullivan is accused Somali medd in\n",
      "A car with a cheaper sticker price can often cost consumers | more in the long run . Report based on comparison | Hillsmore in Mogadishu long run . Report based on comparison\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:23,  6.78s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 62 train loss: 0.013508581334789252\n",
      "Businessman Blake Jones seeks to develop renewable energy sources . Jones | ' passion for solar sparked a radical career shift during | ' passion for solar sparked a radical career shift during\n",
      "Sixteen U.S. embassies in Europe receive mail containing white powder | . Tests show powder harmless in 15 cases; results pen | . Washingtons show powder harmless in 15 cases; results pen\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:02,  7.47s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 63 train loss: 0.010697818528218515\n",
      "Peter Bregman: In the boom, people sacrific | ed their joys for more money . He says bey | ed the jo joys for more money . He says ty\n",
      "Jackson family has been in spotlight along with Michael . Father  | Joseph was a sometimes rough taskmaster . Sibling | Joseph was a sometimes rough taskmaster . Sibling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [17:38,  7.30s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 64 train loss: 0.009684301604484689\n",
      "University of Memphis basketball has brought together whites, | blacks, fan says . Memphis team has a chance to do | blacks, fan says . Memphist team has a full to do\n",
      "Axed Chrysler dealership holds central place in tiny | Georgia crossroads . Generations of families have purchased vehicle | Georgia crossroads . Lerations of families have purchased vehicle\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:44,  3.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 65 train loss: 0.00993766696352897\n",
      "Immigration raid led to detention of 28 people in Washington state . | Raid was first of its kind under the new Obama administration . Homeland | Raid was first of its kind under the Obama Mar administration . Homeland\n",
      "New Yorker article says Congress authorized up to $400 million for cover | t ops in Iran . Journalist Seymour Hersh says | tsifs in Iran . Journalist Sey Womanour Her hav says\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:57,  2.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 66 train loss: 0.007584977988153696\n",
      "NEW: British counter-terrorism experts re-in | spect Bhutto's vehicle . NEW: Bhutto's assassination was her own | spect Bhutto's vehicleing NEW: Bhutto wass assassination was her own\n",
      "NEW: \"There's no cause of panic,\" says New Del | hi police spokesman . Security raised at all the nation's airports | hi police spokesman . Security raised at all the nation's airports\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:49,  6.96s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67 train loss: 0.006443165361495882\n",
      "Newer Acura MD | SUV, but it's a deluxe version . BM | SUV, but it's a deluxe version . BM\n",
      "Aung San Suu Kyi says she's committed to pursuing | a dialogue with the ruling junta . Suu Kyi meets with three | a dialogue with the ruling junta . Suu Kyi meets with three\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [17:25,  7.21s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 68 train loss: 0.004655806566909726\n",
      "Seizures in teenagers can be caused by dozens of disorders | , genetics . Seizure said to have caused the death of | , genetics . Seizure said to have caused the death of\n",
      "CNN journalist in Denver, Colorado, weighs in on how economy affect | ing city . UPS store owner says business down at least 25 | ing city . UPS store owners business down at least 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:20,  7.59s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 69 train loss: 0.006930545105695211\n",
      "Michael Lamar strikes an Obama-like figure in ads for | a bank in Turkey . Lamar worked for JP Morgan Chas | a bank in Turkey . Lamar worked for JP Morgan Chas\n",
      "Heidi Newfield is up for five Academy of Country Music A | wards . Newfield pursuing solo career after being lead singer | wards . Newfield pursuing solo career after leading lead singer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:23,  7.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 70 train loss: 0.00577883155342063\n",
      "Australian Tim Cahill denies Liverpool win with late goal as Ever | ton draw 1-1 . Liverpool had grabbed the lead when Steven | ton draw 1-1 . Liverpool had grabbed the lead when Steven\n",
      "More than 340 people quarantined in Hong Kong follow | ing single case of H1N1 . Those isolated due to | ing single case of H1N1 . Hohose isolated due to\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.18s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.6070117354393005\n",
      "Smartwatches have hidden, darker side the companies selling | them are unlikely to talk about, says Andre Spicer . | s . Car-Rahalons infects in UA\n",
      "Timothy Stanley: GOP senators' letter to Iranian leaders seem | s extraordinary . But undermining a president's foreign policy | some their police . Court: Police: Ga police the two politician politician\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:31,  6.84s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71 train loss: 0.007406085699090156\n",
      "Mom thinks girl was abused while in the care of a baby | sitter, attorney says . Mother had no idea daughter had been abuse | sitter, attorney says . Mother had no idea daughter had been abuse\n",
      "Expert: Many looking for happiness don't realize they | already have it . Funeral director finds happiness in job | already have it . Funeral director finds happiness in job\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:08,  7.50s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72 train loss: 0.006344721427765386\n",
      "Chris Shurn served four years in San  | . Shurn earned a GED and nearly completed | . Hehurn earned a fullE complete and nearly completed\n",
      "Scotland Yard releases report into assassination of Benazir Bhutto . On | ly apparent injury was a major trauma to the right side of the | ly apparent injury was a major traums to the right side of the\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:25,  7.62s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 73 train loss: 0.005207455572511615\n",
      "David Hawkins: Admission tests are wrongly used to | rank college quality . Hawkins says Baylor University's in | rank college quality . Hawkins says Baylor University's in\n",
      "Officials, academics gather to discuss North Korea's energy need | s . Diplomacy was part of the discussion at Georgia | s . Diplomacy was part of the discussion at Georgia\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 74 train loss: 0.005376541828094371\n",
      "Begala: McCain's VP choice unqualified to be heart | beat from the presidency . Choice of Alaska Gov. | beat from the presidency . Choice su Alaska Gov.\n",
      "Pakistan: Fighting began after Indian soldiers crossed the Line of Con | trol . India accused Pakistan of attacking one of its patrol | trol . India accused Pakistan of attacking one of its patrol\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [15:44,  6.51s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 75 train loss: 0.004152925698696797\n",
      "Sheikha Lubna was first female minister in the United Arab Emirates . | Openness to foreign ownership is the \"natural path | Openness to foreign ownership is the \"natural path\n",
      "Winter storm warnings issued from New England south to Georgia, Alabama | . Washington declares snow emergency, could get 10 inches by Monday | . Washington declares snow emergency, could make 10 inche in by Monday\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [11:42,  4.85s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76 train loss: 0.004000388553912013\n",
      "NEW: Stephen Morgan, 29, arrested in university student's | shooting death . Suspect's sister earlier urged him to turn himself | shooting death . E's sister earlier urged him to turn himself\n",
      "University students at ETH Zurich have designed and built a | robotic boat . The boat needs no sailors and uses  | robotic boat . The boat needs no sailors and uses \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:09,  6.68s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 77 train loss: 0.005044551196122735\n",
      "Bernard Kerik is accused of failing to report more than $500 | ,000 in income . Kerik is the former New York City police | ,000 in in Afghanistan . Kerik is the former New York City police\n",
      "Sri Lankan team agreed to replace India in Pakistan after Mumbai attack | s . International teams have long expressed concern about security in Pakistan . | s . International teams have long expressed concern about security in Pakistan .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:21,  6.77s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 78 train loss: 0.0038044377398709283\n",
      "Justice Ruth Bader Ginsburg expects to return to the be | nch by February 23 . The only woman on the Supreme Court | nch by life 23 . Command only woman on the Supreme Court\n",
      "Italian mountaineer Reinhold Messner has inspired others to seek | adventure . Psychologist Dr. James Thompson says adventurers tend | adventure . Psychologist Dr. James Thompson says adventurers tend\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 79 train loss: 0.002624312272809189\n",
      "Richard Cooey, set to die Tuesday, has exhausted | most state, federal appeals . Justices are expected to decide | most state, federal appeals . Justices are expected to decide\n",
      "Analysts say the race will be tight . Some say the Hezbollah- | dominated alliance may win a parliamentary majority . Turn | dominated alliance may win a parliamentary majority . \"urn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [18:19,  7.58s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 80 train loss: 0.005865099048241973\n",
      "NEW: NTSB says \"very little\" remains found in wreckage . NTSB | : It could take \"weeks, perhaps months\" to | : It could take \"weeks,hahaps months\" to\n",
      "No one submitted minimum $3.2 million bid for Michael | Vick's mansion . Vick, former quarterback of the | Vick's mansion . Vick, married quarterback of the\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.22s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.6297488808631897\n",
      "Parents of teens should have a criminal defense lawyer in mind  | -- just in case, says author . Author Lisa Green calls this the | C . Ecks, who was in exposure also and accompani\n",
      "Media coverage of ISIS could spur \"real | wave of Islamophobia,\" report's edit | ,\" for December . Judge he could lifts join spread its\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [17:13,  7.13s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 81 train loss: 0.0051807226275960945\n",
      "We ask, should humanity expand into space | Space movement say space is humanity's future . But others say | Space movement say space is humanity's future . But others say\n",
      "Terkel won Pulitzer Prize in 1985 for book about World War | II, \"The Good War\" Son: \"My dad led | II, \"The Good War\" Son: \"My dad led\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 82 train loss: 0.00429867510202116\n",
      "President Bush plans to ask Congress to lift offshore drilling ban Wednesday | . McCain says he opposes ban; states should decide . Curr | . McCain oppose he opposes ban; states should decide . Curr\n",
      "\"Watchmen\" premiered in more theaters than any other R | -rated movie in history . Tyler Perry's \"Mad | -rated movie in history . Tyler Perry \"s \"Mad early\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [15:56,  6.60s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 83 train loss: 0.004926049444374853\n",
      "Actress Natasha Richardson fell on a beginners' trail in  |  | ueuebec, Canada . Actress had by \"visible\n",
      "Anh \"Joseph\" Cao is first Vietnamese-American elected to | the U.S. House . Cao, of Louisiana, defeated | the U.S. House . Cad, of Louisiana, defeated\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 84 train loss: 0.004546508877056426\n",
      "Most common car safety system is ABS or anti-lock | brake system . EBD: subsystem of | brake system . EBD: subsystemial\n",
      "When a business scam fails, it tends to fail in | rather grand fashion . 360 B.C. scam: S | rather grand fashion . 360 .. scam. scam: S\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:10,  6.69s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 85 train loss: 0.005084355458906242\n",
      "Ukrainian-born John Demjanjuk loses an appeal to avoid deportation . German | authorities seek him for alleged involvement in Nazi camp killings . The retire | authorities seek him for alleged involvement in Nazi to ins . The murder\n",
      "London football club Chelsea appoint Carlo Ancelotti as their | new manager . Ancelotti has been in charge of Italian giant | new manager . Anselotti has been in charge of Italian giant\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:22,  6.78s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 86 train loss: 0.007442781810873541\n",
      "Jet's left engine fell off as plane traveled between 25 | 0 and 300 kmh, reports say . No traces of | 0 and 300 kmh, reports say . No traces of\n",
      "Laborers in Palisades Park say two days of | work at $90 | work at $90mostday is a  . . They hope\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [03:48,  1.58s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 87 train loss: 0.008169963888438611\n",
      "Thousands of items sent from around globe are housed on Tech | campus . Items range from letters from kids | campus . Items range from letters from kids\n",
      "A special memorial has marked the 20th anniversary of Hills | borough tragedy . 96 Liverpool supporters died in crush at | borough orough . 96 Liverpool supporters died in crush at\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [15:53,  6.58s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 88 train loss: 0.005615780794800356\n",
      "Woman travels from Kenya to India for a tricky third kidn | ey transplant . Lorna Irungu suffers from l | ey transplant . Lorna Irungu suffers from l\n",
      "Richardson's death raises question: When should you go to E | R after head injury | R after head injury,zzzziness, vomitings\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 89 train loss: 0.004985615005716681\n",
      "Casey Anthony's defense team has big challenge, experts | say . Lack of cause of death, physical evidence could hinder prosecution | say . Lack of cause of death, physical evidence could hinder prosecution\n",
      "FIFA will announce its Player of the Year for 2008 in Zurich Monday evening | . Cristiano Ronaldo has been tipped as a favorite to win | . Cristiano Ronaldo has been tipped as a favorite to win\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:24,  6.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 90 train loss: 0.007083743043115427\n",
      "Carly Fiorina: It's understandable that people are outraged | at Wall St. pay . Fiorina says government shouldn't set | at Wall St. pay . Fiorina says government shouldn't set\n",
      "Ten-minute message delivered by Ayman al-Zawahiri, a | native of Egypt . In address, al-Zawahiri says violence a \" | native of Egypt . Inever, who-Zawahiri a violence a \"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:02,  1.26s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid loss: 0.644891619682312\n",
      "Smartwatches have hidden, darker side the companies selling | them are unlikely to talk about, says Andre Spicer . | s . White House: President Bush-goal Presidenth re politically\n",
      "Indian broadcaster Puthiya Thalaimurai d | rew protests for a show about traditional necklaces worn | . don't, his mother, don't, his mother .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [16:24,  6.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 91 train loss: 0.006986500144582884\n",
      "Habitual felon likely to avoid charges in presumed murders | of four people . Scott Kimball's plea deal includes revealing | of four people . Scott Kimball's plea deal includes revealing\n",
      "Janelle Monae popping up on commercial, on tour | , on TV . Singer has alter ego: Cindi Maywea | on on TV ship Singer has alter ne: Cind: Maycrafta\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [17:07,  7.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 92 train loss: 0.005264191368016704\n",
      "President Cristina Fernandez de Kirchner says addicts should not | be persecuted . Argentine government wants Congress to pass | be persecuted . Argentine government wants Congress to pass\n",
      "Anti-election demonstrators, Indian security forces clash during state | elections . Police say they used batons to restore order, and | elections . S say they used batons to Islamore orders and\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [15:51,  6.56s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 93 train loss: 0.004917647458356002\n",
      "Vimlendu takes a group of children to see a village | in Uttaranchal . His aim was to show them the differen | in Uttaranchal . His aim was to show them the differen\n",
      "Eight Florida teens to be tried as adults in videotaped beating case | . Video shows 16-year-old girl punched by | . Video shows 16-year-old girl punched by\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [12:34,  5.20s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 94 train loss: 0.00309387323735603\n",
      "Alleged pirate known in official documents as \"Pirate Defendant\" Alleg | ed pirate brought to Djibouti aboard the USNS | ed pirate brought to Djibouti aboard the USNS\n",
      "Larry Zeiger moved from Brooklyn to Miami | in 1957 in pursuit of a radio hosting gig . | in 1957 in pursuo of a radio .ing gig .\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:45,  3.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 95 train loss: 0.0036947479054074864\n",
      "\"Flashpackers\" are professionals older than 30 who | prefer hostels to hotels . Hostels in New York, | preser hostels to hotels . Hostels in New York,\n",
      "Harrison's star next to Hollywood headquarters of Capitol | Records . Harrison, who died of cancer in 2001, is second | Records . Harrison, who died of cancer in 2001, is second\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 96 train loss: 0.002400477565343267\n",
      "Andy Roddick forced to retire from his  | 's Club semifinal with ankle injury . The injury means | 's Club semifinal with ankle injury . The injury means\n",
      "First Principal Voices round table debate takes place in Doha | ,  | , thatar . Olafur Grimsson states \"it\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:43,  3.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 97 train loss: 0.002656286178509994\n",
      "Compact-camera makers are testing the waters with high- | end features like GPS . Premium features like high-speed | end features like GPS . Premier features like high-speed\n",
      "The company has become a huge name in communications in just 20 | years .  | years . moreualcomm has a portfolio 10 app\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 98 train loss: 0.0026482165935609874\n",
      "Dharavi, in the heart of Mumbai, is one of the | biggest slums in the world . The massive redev | biggest s s in the world . The massive redev\n",
      "Justice Department must decide whether to pursue investigation . Former agent | John Kiriakou talked about interrogation technique on TV | John Kiriakou talked about interrogation technique on TV\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:42,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 99 train loss: 0.004156963520779692\n",
      "Ruben Navarrette: Hate crimes should be punished more severely . | He says hate crimes terrorize society as a whole . Navarrette | He says hate a terrorize society as a who call . Navarrette\n",
      "Toobin: Ruling falls in line with recent court decision | s on death penalty . Justice Kennedy, in decision, said | s on death penalty . Justice Kennedy, in decision, said\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 100\n",
    "DISPLAY_BATCHES = 2\n",
    "OUT_SEQUENCE_LEN = wrapper.y_length\n",
    "PRINT_VALID = True\n",
    "ACCUMULATE_STEPS = 1\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    # Run over the training examples\n",
    "    train_loss = 0\n",
    "    match_pct = 0\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    start = time.time()\n",
    "    for batch, (sequence, target, prev_target) in tqdm(enumerate(train)):\n",
    "        pred = model(sequence.to(DEVICE))\n",
    "\n",
    "        # If you use a batch, need to reshape pred to be batch * sequence, embedding_len to be compatible\n",
    "        # Similar reshape with target to be batch * sequence vector of class indices\n",
    "        cpred = pred[:, wrapper.x_length:]\n",
    "        ctarget = target[:, wrapper.x_length:]\n",
    "        loss = loss_fn(cpred.reshape(-1, cpred.shape[-1]), ctarget.reshape(-1).to(DEVICE))\n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "\n",
    "        # Accumulate gradients\n",
    "        # This seems to perform worse than no accumulation over a\n",
    "        # small data set.  Test with larger set.\n",
    "        if batch % ACCUMULATE_STEPS == 0:\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad(set_to_none=True)\n",
    "    end = time.time()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        mean_loss = train_loss / len(train) / BATCH_SIZE\n",
    "        wandb.log({\"loss\": mean_loss, \"epoch_time\": end - start})\n",
    "        print(f\"Epoch {epoch} train loss: {mean_loss}\")\n",
    "        sents = generate(sequence, pred, target, wrapper)\n",
    "        for sent in sents[:DISPLAY_BATCHES]:\n",
    "            print(sent)\n",
    "\n",
    "        if PRINT_VALID and epoch % 10 ==0:\n",
    "            # Compute validation loss.  Unless you have a lot of training data, the validation loss won't decrease.\n",
    "            valid_loss = 0\n",
    "            # Deactivate dropout layers\n",
    "            model.eval()\n",
    "            for batch, (sequence, target, prev_target) in tqdm(enumerate(valid)):\n",
    "                # Inference token by tokens\n",
    "                outputs = sequence[:,:(wrapper.x_length + 1)].to(DEVICE)\n",
    "                # TODO: Investigate memory leak with valid generation\n",
    "                for i in range(OUT_SEQUENCE_LEN):\n",
    "                    pred = model(outputs)\n",
    "                    last_output = torch.argmax(pred, dim=2)\n",
    "                    outputs = torch.cat((outputs, last_output[:,-1:]), dim=1)\n",
    "\n",
    "                cpred = pred[:, wrapper.x_length:]\n",
    "                ctarget = target[:, wrapper.x_length:]\n",
    "                loss = loss_fn(cpred.reshape(-1, cpred.shape[-1]), ctarget.reshape(-1).to(DEVICE))\n",
    "                valid_loss += loss.item()\n",
    "            mean_loss = valid_loss / len(valid) / BATCH_SIZE\n",
    "            wandb.log({\"valid_loss\": mean_loss})\n",
    "            print(f\"Valid loss: {mean_loss}\")\n",
    "            sents = generate(sequence, pred, target, wrapper)\n",
    "            for sent in sents[:DISPLAY_BATCHES]:\n",
    "                print(sent)\n",
    "            # Reactivate dropout\n",
    "            model.train()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================================================\n",
      "Layer (type:depth-idx)                   Param #\n",
      "=================================================================\n",
      "Transformer                              5,121,024\n",
      "├─ModuleList: 1-1                        --\n",
      "│    └─Dropout: 2-1                      --\n",
      "│    └─Dropout: 2-2                      --\n",
      "├─ModuleList: 1-2                        --\n",
      "│    └─DecoderBlock: 2-3                 --\n",
      "│    │    └─MultiHeadAttention: 3-1      589,824\n",
      "│    │    └─ModuleList: 3-2              --\n",
      "│    │    └─ModuleList: 3-3              2,048\n",
      "│    │    └─SwiGLU: 3-4                  3,145,728\n",
      "│    └─DecoderBlock: 2-4                 --\n",
      "│    │    └─MultiHeadAttention: 3-5      589,824\n",
      "│    │    └─ModuleList: 3-6              --\n",
      "│    │    └─ModuleList: 3-7              2,048\n",
      "│    │    └─SwiGLU: 3-8                  3,145,728\n",
      "│    └─DecoderBlock: 2-5                 --\n",
      "│    │    └─MultiHeadAttention: 3-9      589,824\n",
      "│    │    └─ModuleList: 3-10             --\n",
      "│    │    └─ModuleList: 3-11             2,048\n",
      "│    │    └─SwiGLU: 3-12                 3,145,728\n",
      "│    └─DecoderBlock: 2-6                 --\n",
      "│    │    └─MultiHeadAttention: 3-13     589,824\n",
      "│    │    └─ModuleList: 3-14             --\n",
      "│    │    └─ModuleList: 3-15             2,048\n",
      "│    │    └─SwiGLU: 3-16                 3,145,728\n",
      "│    └─DecoderBlock: 2-7                 --\n",
      "│    │    └─MultiHeadAttention: 3-17     589,824\n",
      "│    │    └─ModuleList: 3-18             --\n",
      "│    │    └─ModuleList: 3-19             2,048\n",
      "│    │    └─SwiGLU: 3-20                 3,145,728\n",
      "│    └─DecoderBlock: 2-8                 --\n",
      "│    │    └─MultiHeadAttention: 3-21     589,824\n",
      "│    │    └─ModuleList: 3-22             --\n",
      "│    │    └─ModuleList: 3-23             2,048\n",
      "│    │    └─SwiGLU: 3-24                 3,145,728\n",
      "=================================================================\n",
      "Total params: 27,546,624\n",
      "Trainable params: 27,546,624\n",
      "Non-trainable params: 0\n",
      "=================================================================\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from torchinfo import summary\n",
    "\n",
    "print(summary(model))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "STAGE:2023-01-30 07:12:17 1952:4665970 ActivityProfilerController.cpp:294] Completed Stage: Warm Up\n",
      "STAGE:2023-01-30 07:12:17 1952:4665970 ActivityProfilerController.cpp:300] Completed Stage: Collection\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Transformer.forward() takes 2 positional arguments but 3 were given",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[12], line 4\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mprofiler\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m profile, record_function, ProfilerActivity\n\u001B[1;32m      3\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m profile(activities\u001B[38;5;241m=\u001B[39m[ProfilerActivity\u001B[38;5;241m.\u001B[39mCPU], record_shapes\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, ) \u001B[38;5;28;01mas\u001B[39;00m prof:\n\u001B[0;32m----> 4\u001B[0m     \u001B[43mmodel\u001B[49m\u001B[43m(\u001B[49m\u001B[43msequence\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mto\u001B[49m\u001B[43m(\u001B[49m\u001B[43mDEVICE\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mprev_target\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mto\u001B[49m\u001B[43m(\u001B[49m\u001B[43mDEVICE\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m      6\u001B[0m \u001B[38;5;28mprint\u001B[39m(prof\u001B[38;5;241m.\u001B[39mkey_averages()\u001B[38;5;241m.\u001B[39mtable(sort_by\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcpu_time_total\u001B[39m\u001B[38;5;124m\"\u001B[39m, row_limit\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m))\n",
      "File \u001B[0;32m~/.virtualenvs/nnets/lib/python3.10/site-packages/torch/nn/modules/module.py:1194\u001B[0m, in \u001B[0;36mModule._call_impl\u001B[0;34m(self, *input, **kwargs)\u001B[0m\n\u001B[1;32m   1190\u001B[0m \u001B[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001B[39;00m\n\u001B[1;32m   1191\u001B[0m \u001B[38;5;66;03m# this function, and just call forward.\u001B[39;00m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_forward_pre_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_backward_hooks\n\u001B[1;32m   1193\u001B[0m         \u001B[38;5;129;01mor\u001B[39;00m _global_forward_hooks \u001B[38;5;129;01mor\u001B[39;00m _global_forward_pre_hooks):\n\u001B[0;32m-> 1194\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mforward_call\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;28;43minput\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1195\u001B[0m \u001B[38;5;66;03m# Do not call functions when jit is used\u001B[39;00m\n\u001B[1;32m   1196\u001B[0m full_backward_hooks, non_full_backward_hooks \u001B[38;5;241m=\u001B[39m [], []\n",
      "\u001B[0;31mTypeError\u001B[0m: Transformer.forward() takes 2 positional arguments but 3 were given"
     ]
    }
   ],
   "source": [
    "from torch.profiler import profile, record_function, ProfilerActivity\n",
    "\n",
    "with profile(activities=[ProfilerActivity.CPU], record_shapes=True, ) as prof:\n",
    "    model(sequence.to(DEVICE), prev_target.to(DEVICE))\n",
    "\n",
    "print(prof.key_averages().table(sort_by=\"cpu_time_total\", row_limit=10))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from einops import rearrange\n",
    "\n",
    "x = torch.rand(4, 3)\n",
    "x[torch.tensor([0,1,0])]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}