16_nlp_with_rnns_and_attention.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Chapter 16 – Natural Language Processing with RNNs and Attention**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_This notebook contains all the sample code in chapter 16._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table align=\"left\">\n",
    "  <td>\n",
    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/jflanigan/handson-ml2/blob/master/16_nlp_with_rnns_and_attention.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
    "  </td>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "    !pip install -q -U tensorflow-addons\n",
    "    IS_COLAB = True\n",
    "except Exception:\n",
    "    IS_COLAB = False\n",
    "\n",
    "# TensorFlow ≥2.0 is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "if not tf.test.is_gpu_available():\n",
    "    print(\"No GPU was detected. LSTMs and CNNs can be very slow without a GPU.\")\n",
    "    if IS_COLAB:\n",
    "        print(\"Go to Runtime > Change runtime and select a GPU hardware accelerator.\")\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"nlp\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can load the IMDB dataset easily:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_test), (X_valid, y_test) = keras.datasets.imdb.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<sos> this film was just brilliant casting location scenery story'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_index = keras.datasets.imdb.get_word_index()\n",
    "id_to_word = {id_ + 3: word for word, id_ in word_index.items()}\n",
    "for id_, token in enumerate((\"<pad>\", \"<sos>\", \"<unk>\")):\n",
    "    id_to_word[id_] = token\n",
    "\" \".join([id_to_word[id_] for id_ in X_train[0][:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "\n",
    "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['test', 'train'])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = info.splits[\"train\"].num_examples\n",
    "test_size = info.splits[\"test\"].num_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 25000)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_size, test_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review: This was soul-provoking! I am an Iranian, and living in th 21st century, I didn't know that such big tribes have been living in such conditions at the time of my grandfather!<br /><br />You see that t ...\n",
      "Label: 1 = Positive\n",
      "\n",
      "Review: A very close and sharp discription of the bubbling and dynamic emotional world of specialy one 18year old guy, that makes his first experiences in his gay love to an other boy, during an vacation with ...\n",
      "Label: 1 = Positive\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for X_batch, y_batch in datasets[\"train\"].batch(2).take(1):\n",
    "    for review, label in zip(X_batch.numpy(), y_batch.numpy()):\n",
    "        print(\"Review:\", review.decode(\"utf-8\")[:200], \"...\")\n",
    "        print(\"Label:\", label, \"= Positive\" if label else \"= Negative\")\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(X_batch, y_batch):\n",
    "    X_batch = tf.strings.substr(X_batch, 0, 300)\n",
    "    X_batch = tf.strings.regex_replace(X_batch, rb\"<br\\s*/?>\", b\" \")\n",
    "    X_batch = tf.strings.regex_replace(X_batch, b\"[^a-zA-Z']\", b\" \")\n",
    "    X_batch = tf.strings.split(X_batch)\n",
    "    return X_batch.to_tensor(default_value=b\"<pad>\"), y_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: id=235, shape=(2, 57), dtype=string, numpy=\n",
       " array([[b'This', b'was', b'soul', b'provoking', b'I', b'am', b'an',\n",
       "         b'Iranian', b'and', b'living', b'in', b'th', b'st', b'century',\n",
       "         b'I', b\"didn't\", b'know', b'that', b'such', b'big', b'tribes',\n",
       "         b'have', b'been', b'living', b'in', b'such', b'conditions',\n",
       "         b'at', b'the', b'time', b'of', b'my', b'grandfather', b'You',\n",
       "         b'see', b'that', b'today', b'or', b'even', b'in', b'on', b'one',\n",
       "         b'side', b'of', b'the', b'world', b'a', b'lady', b'or', b'a',\n",
       "         b'baby', b'could', b'have', b'everything', b'served', b'for',\n",
       "         b'hi'],\n",
       "        [b'A', b'very', b'close', b'and', b'sharp', b'discription', b'of',\n",
       "         b'the', b'bubbling', b'and', b'dynamic', b'emotional', b'world',\n",
       "         b'of', b'specialy', b'one', b'year', b'old', b'guy', b'that',\n",
       "         b'makes', b'his', b'first', b'experiences', b'in', b'his',\n",
       "         b'gay', b'love', b'to', b'an', b'other', b'boy', b'during',\n",
       "         b'an', b'vacation', b'with', b'a', b'part', b'of', b'his',\n",
       "         b'family', b'I', b'liked', b'this', b'film', b'because', b'of',\n",
       "         b'his', b'extremly', b'clear', b'and', b'surrogated', b'sto',\n",
       "         b'<pad>', b'<pad>', b'<pad>', b'<pad>']], dtype=object)>,\n",
       " <tf.Tensor: id=128, shape=(2,), dtype=int64, numpy=array([1, 1])>)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(X_batch, y_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "vocabulary = Counter()\n",
    "for X_batch, y_batch in datasets[\"train\"].batch(32).map(preprocess):\n",
    "    for review in X_batch:\n",
    "        vocabulary.update(list(review.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(b'<pad>', 214077), (b'the', 61137), (b'a', 38564)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary.most_common()[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53893"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 10000\n",
    "truncated_vocabulary = [\n",
    "    word for word, count in vocabulary.most_common()[:vocab_size]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n",
      "12\n",
      "11\n",
      "10000\n"
     ]
    }
   ],
   "source": [
    "word_to_id = {word: index for index, word in enumerate(truncated_vocabulary)}\n",
    "for word in b\"This movie was faaaaaantastic\".split():\n",
    "    print(word_to_id.get(word) or vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = tf.constant(truncated_vocabulary)\n",
    "word_ids = tf.range(len(truncated_vocabulary), dtype=tf.int64)\n",
    "vocab_init = tf.lookup.KeyValueTensorInitializer(words, word_ids)\n",
    "num_oov_buckets = 1000\n",
    "table = tf.lookup.StaticVocabularyTable(vocab_init, num_oov_buckets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=126936, shape=(1, 4), dtype=int64, numpy=array([[   22,    12,    11, 10053]])>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.lookup(tf.constant([b\"This movie was faaaaaantastic\".split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_words(X_batch, y_batch):\n",
    "    return table.lookup(X_batch), y_batch\n",
    "\n",
    "train_set = datasets[\"train\"].repeat().batch(32).map(preprocess)\n",
    "train_set = train_set.map(encode_words).prefetch(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[    6    98     9 ...     0     0     0]\n",
      " [  185     2 10865 ...     0     0     0]\n",
      " [  719  2410  5630 ...     0     0     0]\n",
      " ...\n",
      " [    6    94    13 ...     0     0     0]\n",
      " [   14   498    16 ...     0     0     0]\n",
      " [  168     1  1633 ...     0     0     0]], shape=(32, 64), dtype=int64)\n",
      "tf.Tensor([1 1 1 1 1 1 1 1 0 0 1 1 0 1 1 1 0 1 1 1 0 0 1 1 1 1 1 1 0 1 0 0], shape=(32,), dtype=int64)\n"
     ]
    }
   ],
   "source": [
    "for X_batch, y_batch in train_set.take(1):\n",
    "    print(X_batch)\n",
    "    print(y_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "781/781 [==============================] - 102s 131ms/step - loss: 0.5378 - accuracy: 0.7238\n",
      "Epoch 2/5\n",
      "781/781 [==============================] - 87s 111ms/step - loss: 0.3485 - accuracy: 0.8567\n",
      "Epoch 3/5\n",
      "781/781 [==============================] - 87s 111ms/step - loss: 0.1877 - accuracy: 0.9332\n",
      "Epoch 4/5\n",
      "781/781 [==============================] - 87s 111ms/step - loss: 0.1236 - accuracy: 0.9573\n",
      "Epoch 5/5\n",
      "781/781 [==============================] - 87s 111ms/step - loss: 0.0964 - accuracy: 0.9667\n"
     ]
    }
   ],
   "source": [
    "embed_size = 128\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size,\n",
    "                           mask_zero=True, # not shown in the book\n",
    "                           input_shape=[None]),\n",
    "    keras.layers.GRU(128, return_sequences=True),\n",
    "    keras.layers.GRU(128),\n",
    "    keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "])\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "history = model.fit(train_set, steps_per_epoch=train_size // 32, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or using manual masking:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "781/781 [==============================] - 102s 131ms/step - loss: 0.5436 - accuracy: 0.7172\n",
      "Epoch 2/5\n",
      "781/781 [==============================] - 88s 113ms/step - loss: 0.3519 - accuracy: 0.8564\n",
      "Epoch 3/5\n",
      "781/781 [==============================] - 87s 111ms/step - loss: 0.1950 - accuracy: 0.9306\n",
      "Epoch 4/5\n",
      "781/781 [==============================] - 87s 111ms/step - loss: 0.1226 - accuracy: 0.9579\n",
      "Epoch 5/5\n",
      "781/781 [==============================] - 86s 110ms/step - loss: 0.0922 - accuracy: 0.9679\n"
     ]
    }
   ],
   "source": [
    "K = keras.backend\n",
    "embed_size = 128\n",
    "inputs = keras.layers.Input(shape=[None])\n",
    "mask = keras.layers.Lambda(lambda inputs: K.not_equal(inputs, 0))(inputs)\n",
    "z = keras.layers.Embedding(vocab_size + num_oov_buckets, embed_size)(inputs)\n",
    "z = keras.layers.GRU(128, return_sequences=True)(z, mask=mask)\n",
    "z = keras.layers.GRU(128)(z, mask=mask)\n",
    "outputs = keras.layers.Dense(1, activation=\"sigmoid\")(z)\n",
    "model = keras.models.Model(inputs=[inputs], outputs=[outputs])\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "history = model.fit(train_set, steps_per_epoch=train_size // 32, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reusing Pretrained Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "TFHUB_CACHE_DIR = os.path.join(os.curdir, \"my_tfhub_cache\")\n",
    "os.environ[\"TFHUB_CACHE_DIR\"] = TFHUB_CACHE_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_hub as hub\n",
    "\n",
    "model = keras.Sequential([\n",
    "    hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\",\n",
    "                   dtype=tf.string, input_shape=[], output_shape=[50]),\n",
    "    keras.layers.Dense(128, activation=\"relu\"),\n",
    "    keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "])\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./my_tfhub_cache/82c4aaf4250ffb09088bd48368ee7fd00e5464fe.descriptor.txt\n",
      "./my_tfhub_cache/82c4aaf4250ffb09088bd48368ee7fd00e5464fe/saved_model.pb\n",
      "./my_tfhub_cache/82c4aaf4250ffb09088bd48368ee7fd00e5464fe/variables/variables.data-00000-of-00001\n",
      "./my_tfhub_cache/82c4aaf4250ffb09088bd48368ee7fd00e5464fe/variables/variables.index\n",
      "./my_tfhub_cache/82c4aaf4250ffb09088bd48368ee7fd00e5464fe/assets/tokens.txt\n"
     ]
    }
   ],
   "source": [
    "for dirpath, dirnames, filenames in os.walk(TFHUB_CACHE_DIR):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirpath, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "781/781 [==============================] - 119s 152ms/step - loss: 0.5499 - accuracy: 0.7230\n",
      "Epoch 2/5\n",
      "781/781 [==============================] - 119s 152ms/step - loss: 0.5133 - accuracy: 0.7486\n",
      "Epoch 3/5\n",
      "781/781 [==============================] - 117s 150ms/step - loss: 0.5078 - accuracy: 0.7518\n",
      "Epoch 4/5\n",
      "781/781 [==============================] - 118s 151ms/step - loss: 0.5042 - accuracy: 0.7540\n",
      "Epoch 5/5\n",
      "781/781 [==============================] - 122s 156ms/step - loss: 0.5010 - accuracy: 0.7574\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "\n",
    "datasets, info = tfds.load(\"imdb_reviews\", as_supervised=True, with_info=True)\n",
    "train_size = info.splits[\"train\"].num_examples\n",
    "batch_size = 32\n",
    "train_set = datasets[\"train\"].repeat().batch(batch_size).prefetch(1)\n",
    "history = model.fit(train_set, steps_per_epoch=train_size // batch_size, epochs=5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}