universal_sentence_encoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "steam_descriptions.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "wqdDv1dm127T",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Steam Descriptions\n",
        "\n",
        "Reference: https://github.com/woctezuma/steam-descriptions"
      ]
    },
    {
      "metadata": {
        "id": "oJkd3Zll7H8B",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Setting"
      ]
    },
    {
      "metadata": {
        "id": "OzAS9BrI1a84",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "2c60e703-8871-4fc6-d44d-18acad2c8ec2"
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "mount_folder = '/content/gdrive'\n",
        "drive.mount(mount_folder)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "o5vrUA_1130v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201
        },
        "outputId": "46a47d36-5c57-44de-e215-48c80852168a"
      },
      "cell_type": "code",
      "source": [
        "%cd '/content/gdrive/My Drive/'\n",
        "!rm -rf steam-descriptions/\n",
        "!git clone https://github.com/woctezuma/steam-descriptions.git\n",
        "%cd steam-descriptions/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/My Drive\n",
            "Cloning into 'steam-descriptions'...\n",
            "remote: Enumerating objects: 26, done.\u001b[K\n",
            "remote: Counting objects: 100% (26/26), done.\u001b[K\n",
            "remote: Compressing objects: 100% (23/23), done.\u001b[K\n",
            "remote: Total 592 (delta 10), reused 10 (delta 3), pack-reused 566\u001b[K\n",
            "Receiving objects: 100% (592/592), 69.79 MiB | 13.49 MiB/s, done.\n",
            "Resolving deltas: 100% (343/343), done.\n",
            "Checking out files: 100% (18/18), done.\n",
            "/content/gdrive/My Drive/steam-descriptions\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "UVzKy0cr2Ncp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 717
        },
        "outputId": "695b663d-19b3-423b-e437-35f6e0f26954"
      },
      "cell_type": "code",
      "source": [
        "!pip install -r requirements.txt"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: cython==0.29.6 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 1)) (0.29.6)\n",
            "Requirement already satisfied: gensim==3.7.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 2)) (3.7.1)\n",
            "Requirement already satisfied: pyemd==0.5.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 3)) (0.5.1)\n",
            "Requirement already satisfied: numpy==1.16.2 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 4)) (1.16.2)\n",
            "Requirement already satisfied: sklearn==0.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 5)) (0.0)\n",
            "Requirement already satisfied: spacy==2.0.18 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 6)) (2.0.18)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim==3.7.1->-r requirements.txt (line 2)) (1.1.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim==3.7.1->-r requirements.txt (line 2)) (1.11.0)\n",
            "Requirement already satisfied: smart-open>=1.7.0 in /usr/local/lib/python3.6/dist-packages (from gensim==3.7.1->-r requirements.txt (line 2)) (1.8.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn==0.0->-r requirements.txt (line 5)) (0.20.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (1.0.2)\n",
            "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (2.0.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (2.18.4)\n",
            "Requirement already satisfied: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (1.35)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (2.0.2)\n",
            "Requirement already satisfied: dill<0.3,>=0.2 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (0.2.9)\n",
            "Requirement already satisfied: regex==2018.01.10 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (2018.1.10)\n",
            "Requirement already satisfied: thinc<6.13.0,>=6.12.1 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (6.12.1)\n",
            "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy==2.0.18->-r requirements.txt (line 6)) (0.9.6)\n",
            "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (2.49.0)\n",
            "Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (0.98)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (1.9.106)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.0.18->-r requirements.txt (line 6)) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.0.18->-r requirements.txt (line 6)) (2018.11.29)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.0.18->-r requirements.txt (line 6)) (2.6)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.0.18->-r requirements.txt (line 6)) (1.22)\n",
            "Requirement already satisfied: wrapt<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy==2.0.18->-r requirements.txt (line 6)) (1.10.11)\n",
            "Requirement already satisfied: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy==2.0.18->-r requirements.txt (line 6)) (0.9.0.1)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy==2.0.18->-r requirements.txt (line 6)) (4.28.1)\n",
            "Requirement already satisfied: msgpack<0.6.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy==2.0.18->-r requirements.txt (line 6)) (0.5.6)\n",
            "Requirement already satisfied: msgpack-numpy<0.4.4 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy==2.0.18->-r requirements.txt (line 6)) (0.4.3.2)\n",
            "Requirement already satisfied: botocore<1.13.0,>=1.12.106 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (1.12.106)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (0.2.0)\n",
            "Requirement already satisfied: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc<6.13.0,>=6.12.1->spacy==2.0.18->-r requirements.txt (line 6)) (0.9.0)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.106->boto3->smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (2.5.3)\n",
            "Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.106->boto3->smart-open>=1.7.0->gensim==3.7.1->-r requirements.txt (line 2)) (0.14)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LQ3FgyRI7K59",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Load data"
      ]
    },
    {
      "metadata": {
        "id": "vdVDcqoG6CA_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "87f7d81d-1915-4ba6-91b2-b2501f4cbe86"
      },
      "cell_type": "code",
      "source": [
        "%cd '/content/gdrive/My Drive/steam-descriptions/'\n",
        "!ls data/"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/My Drive/steam-descriptions\n",
            "aggregate_prettyprint.json  README.md  tokens.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "C60L0dAO6gLu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from utils import load_raw_data\n",
        "from gensim.parsing.preprocessing import strip_tags, remove_stopwords\n",
        "from gensim.utils import simple_preprocess"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9x1KHs532_n3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "steam_sentences = load_raw_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5rxWw2P464t4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "pre_processed_steam_sentences = dict()\n",
        "\n",
        "for app_id in steam_sentences:\n",
        "  game_data = steam_sentences[app_id]\n",
        "  \n",
        "  original_str = str(strip_tags(game_data['text']))\n",
        "\n",
        "  original_str = original_str.replace('\\t', ' ')\n",
        "\n",
        "  # Reference: https://nicschrading.com/project/Intro-to-NLP-with-spaCy/\n",
        "  original_str = original_str.strip().replace('\\n', ' ').replace('\\r', ' ')\n",
        "  original_str = original_str.replace('&amp;', 'and').replace('&gt;', '>').replace('&lt;', '<')\n",
        "  \n",
        "  pre_processed_steam_sentences[app_id] = original_str  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ohE4jN6n8Aa1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4ac56a84-5eb5-408f-c67c-48914b258b8d"
      },
      "cell_type": "code",
      "source": [
        "len(pre_processed_steam_sentences)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "30885"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "9kHJ7JaQ2Ytm",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Tensorflow Hub\n",
        "\n",
        "References:\n",
        "-   https://tfhub.dev/google/universal-sentence-encoder/2\n",
        "-   https://colab.research.google.com/github/tensorflow/hub/blob/master/examples/colab/semantic_similarity_with_tf_hub_universal_encoder.ipynb"
      ]
    },
    {
      "metadata": {
        "id": "6kWENY1O2Wzf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Install the latest Tensorflow version.\n",
        "!pip3 install --quiet \"tensorflow>=1.7\"\n",
        "# Install TF-Hub.\n",
        "!pip3 install --quiet tensorflow-hub\n",
        "!pip3 install --quiet seaborn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QQswvlM827fX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "a566b4cc-3405-4c15-d145-d463d9ad1135"
      },
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "import re\n",
        "import seaborn as sns"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0307 21:29:40.314654 140471953299328 __init__.py:56] Some hub symbols are not available because TensorFlow version is less than 1.14\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "0fi8tDvs275V",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder/2\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/2\", \"https://tfhub.dev/google/universal-sentence-encoder-large/3\"]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cij5Mh8U8ttb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "app_id_list = []\n",
        "messages = []\n",
        "for app_id in sorted(pre_processed_steam_sentences, key=int):\n",
        "  app_id_list.append(app_id)\n",
        "  messages.append(pre_processed_steam_sentences[app_id])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "J-vBiDeg29gs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 147
        },
        "outputId": "03c69171-cb41-40a6-ddfd-da65d683ac65"
      },
      "cell_type": "code",
      "source": [
        "# Import the Universal Sentence Encoder's TF Hub module\n",
        "embed = hub.Module(module_url)\n",
        "\n",
        "# Reduce logging output.\n",
        "tf.logging.set_verbosity(tf.logging.ERROR)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/control_flow_ops.py:3632: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0307 21:30:06.900073 140471953299328 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/control_flow_ops.py:3632: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "yRVJnVKDElPB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def chunks(l, n):\n",
        "    \"\"\"Yield successive n-sized chunks from l.\"\"\"\n",
        "    for i in range(0, len(l), n):\n",
        "        yield l[i:i + n]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UHyD_7R-EDp0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "num_elements_per_chunk = 1000\n",
        "\n",
        "for chunk_no, message_chunk in enumerate(chunks(messages, num_elements_per_chunk)):\n",
        "  print('Chunk n°{}'.format(chunk_no))\n",
        "\n",
        "  with tf.Session() as session:\n",
        "    session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
        "    message_embeddings = session.run(embed(message_chunk))\n",
        "\n",
        "    for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):\n",
        "      print(\"Message: {}\".format(message_chunk[i]))\n",
        "      print(\"Embedding size: {}\".format(len(message_embedding)))\n",
        "      message_embedding_snippet = \", \".join(\n",
        "          (str(x) for x in message_embedding[:3]))\n",
        "      print(\"Embedding: [{}, ...]\\n\".format(message_embedding_snippet))\n",
        "      if i>3:\n",
        "        break\n",
        "  \n",
        "  np.save('universal-sentence-encoder-features_'+str(chunk_no)+'.npy', message_embeddings)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3jJA4y-eNMgL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "with open('universal-sentence-encoder-appids.txt', 'w', encoding='utf-8') as f:\n",
        "  print(app_id_list, file=f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kGCKKyiT87To",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 586
        },
        "outputId": "73c47b60-5299-4a66-88a0-2f3fb8fa65d7"
      },
      "cell_type": "code",
      "source": [
        "num_samples = len(messages)\n",
        "num_features = 512\n",
        "\n",
        "message_embeddings = np.zeros((num_samples, num_features))\n",
        "\n",
        "for chunk_no, _ in enumerate(chunks(messages, num_elements_per_chunk)):\n",
        "  current_message_embeddings = np.load('universal-sentence-encoder-features_'+str(chunk_no)+'.npy')\n",
        "  \n",
        "  my_start = chunk_no*num_elements_per_chunk\n",
        "  my_end = min(num_samples, (chunk_no+1)*num_elements_per_chunk)\n",
        "  \n",
        "  message_embeddings[my_start:my_end, :] = current_message_embeddings\n",
        "  print('Chunk n°{}: [{}, {}['.format(chunk_no, my_start, my_end))    \n",
        "  \n",
        "np.save('universal-sentence-encoder-features.npy', message_embeddings)  "
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Chunk n°0: [0, 1000[\n",
            "Chunk n°1: [1000, 2000[\n",
            "Chunk n°2: [2000, 3000[\n",
            "Chunk n°3: [3000, 4000[\n",
            "Chunk n°4: [4000, 5000[\n",
            "Chunk n°5: [5000, 6000[\n",
            "Chunk n°6: [6000, 7000[\n",
            "Chunk n°7: [7000, 8000[\n",
            "Chunk n°8: [8000, 9000[\n",
            "Chunk n°9: [9000, 10000[\n",
            "Chunk n°10: [10000, 11000[\n",
            "Chunk n°11: [11000, 12000[\n",
            "Chunk n°12: [12000, 13000[\n",
            "Chunk n°13: [13000, 14000[\n",
            "Chunk n°14: [14000, 15000[\n",
            "Chunk n°15: [15000, 16000[\n",
            "Chunk n°16: [16000, 17000[\n",
            "Chunk n°17: [17000, 18000[\n",
            "Chunk n°18: [18000, 19000[\n",
            "Chunk n°19: [19000, 20000[\n",
            "Chunk n°20: [20000, 21000[\n",
            "Chunk n°21: [21000, 22000[\n",
            "Chunk n°22: [22000, 23000[\n",
            "Chunk n°23: [23000, 24000[\n",
            "Chunk n°24: [24000, 25000[\n",
            "Chunk n°25: [25000, 26000[\n",
            "Chunk n°26: [26000, 27000[\n",
            "Chunk n°27: [27000, 28000[\n",
            "Chunk n°28: [28000, 29000[\n",
            "Chunk n°29: [29000, 30000[\n",
            "Chunk n°30: [30000, 30885[\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}