Chapter 11 Case Study Cross-Domain Sentiment Adaptation-Book-Kitchen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lu3md6R2IFDD"
   },
   "source": [
    "## Case Study - Cross Domain Sentiment Analysis (Books to Kitchen Appliances) sentiment transfer learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "s1aXLXJ6IFDE"
   },
   "source": [
    "John Blitzer, Mark Dredze, Fernando Pereira. Biographies, Bollywood, Boom-boxes and Blenders: Domain Adaptation for Sentiment Classification. Association of Computational Linguistics (ACL), 2007.\n",
    "The Multi-Domain Sentiment Dataset contains product reviews taken from Amazon.com from 4 product types (domains): Kitchen, Books, DVDs, and Electronics. Each domain has several thousand reviews, but the exact number varies by domain.\n",
    "\n",
    "\n",
    "<img src=\"attachment:Book-Kitchen.png\" width=\"600\", height=\"400\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages and verify versions\n",
    "\n",
    "# numpy\n",
    "import numpy\n",
    "print('numpy: %s' % numpy.__version__)\n",
    "# scipy\n",
    "import scipy\n",
    "print('scipy: %s' % scipy.__version__)\n",
    "# matplotlib\n",
    "import matplotlib\n",
    "print('matplotlib: %s' % matplotlib.__version__)\n",
    "# pandas\n",
    "import pandas\n",
    "print('pandas: %s' % pandas.__version__)\n",
    "# nltk\n",
    "import nltk\n",
    "print('nltk: %s' % nltk.__version__)\n",
    "# scikit-learn\n",
    "import sklearn\n",
    "print('sklearn: %s' % sklearn.__version__)\n",
    "# wordcloud\n",
    "import wordcloud\n",
    "print('wordcloud: %s' % wordcloud.__version__)\n",
    "# keras\n",
    "import keras\n",
    "print('keras: %s' % keras.__version__)\n",
    "# tensorflow\n",
    "import tensorflow\n",
    "print('tensorflow: %s' % tensorflow.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4vdarrclIFDF",
    "outputId": "40cc2e2a-5475-42a6-e46c-84367f50fa4a"
   },
   "outputs": [],
   "source": [
    "# Basic packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import collections\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "# Packages for data preparation\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
    "from nltk.corpus import stopwords\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import average_precision_score, precision_recall_curve, confusion_matrix\n",
    "import itertools\n",
    "\n",
    "# Packages for modeling\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers\n",
    "\n",
    "# Word cloud visualization libraries\n",
    "#from scipy.misc import imresize\n",
    "from PIL import Image\n",
    "from wordcloud import WordCloud, ImageColorGenerator, STOPWORDS\n",
    "from collections import Counter\n",
    "\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "# Packages for Keras\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Flatten, UpSampling1D, Dense, Dropout, AlphaDropout, ThresholdedReLU, Convolution1D, ZeroPadding1D, Activation, MaxPooling1D, SpatialDropout1D, Input\n",
    "from keras.layers import GlobalMaxPooling1D, concatenate, LSTM, Bidirectional, BatchNormalization, Concatenate, Dot, Multiply, RepeatVector\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import np_utils\n",
    "from keras.layers import Bidirectional, TimeDistributed\n",
    "from keras.layers.core import Reshape, Activation\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard\n",
    "from keras.constraints import maxnorm\n",
    "from keras.regularizers import l2\n",
    "\n",
    "# data preprocessing\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download Glove embeddings\n",
    "!mkdir embeddings \n",
    "!cd embeddings; wget http://nlp.stanford.edu/data/glove.6B.zip\n",
    "!cd embeddings; unzip glove.6B.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3oGo1tbxIFDK"
   },
   "source": [
    "## GLOBAL VARIABLES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gOILi1FIIFDL"
   },
   "outputs": [],
   "source": [
    "# parameters for learning\n",
    "NB_WORDS = 15000 # Parameter indicating the number of words we'll put in the dictionary\n",
    "NB_EPOCHS = 20  # Number of epochs\n",
    "BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent\n",
    "MAX_LEN = 300  # Maximum number of words in a sequence\n",
    "GLOVE_DIM = 100  # Number of dimensions of the GloVe word embeddings\n",
    "LABELS_COLUMN_NAME = 'labels'  # labels for reviews\n",
    "REVIEWS_COLUMN_NAME = 'reviews'  # actual text with reviews\n",
    "SOURCE_DOMAIN = 'books'  # source domain from the dataset\n",
    "TARGET_DOMAIN = 'kitchen'  # target doman from the dataset\n",
    "MAX_FEATURES = 10000  # BOW and n-gram based models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SRDDwTRCIFDM"
   },
   "source": [
    "## FUNCTIONS AND UTILITIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Mq6uySwSIFDN"
   },
   "outputs": [],
   "source": [
    "def train_model(\n",
    "        model,\n",
    "        X_train,\n",
    "        y_train,\n",
    "        X_valid=None,\n",
    "        y_valid=None,\n",
    "        number_of_epochs=NB_EPOCHS):\n",
    "    \"\"\"\n",
    "    This function trains the model and evaluates using validation set \n",
    "    or uses 20% of training data with accuracy as the metric.\n",
    "    \"\"\"\n",
    "    start = time.time()\n",
    "    model.compile(\n",
    "        optimizer=Adam(),\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy'])\n",
    "    if(X_valid is not None):\n",
    "        history = model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            epochs=number_of_epochs,\n",
    "            batch_size=BATCH_SIZE,\n",
    "            validation_data=(\n",
    "                X_valid,\n",
    "                y_valid),\n",
    "            verbose=1)\n",
    "    else:\n",
    "        history = model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            epochs=number_of_epochs,\n",
    "            batch_size=BATCH_SIZE,\n",
    "            validation_split=0.2)\n",
    "    end = time.time()\n",
    "    total_time = end - start\n",
    "    print(\"Model Training and Validation time %s secs\" % (total_time))\n",
    "    return history\n",
    "\n",
    "\n",
    "def eval_metric(history, metric_name):\n",
    "\n",
    "    metric = history.history[metric_name]\n",
    "    val_metric = history.history['val_' + metric_name]\n",
    "\n",
    "    e = range(1, NB_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, metric, color='navy', label='Train ' + metric_name)\n",
    "    plt.plot(e, val_metric, color='red', label='Validation ' + metric_name)\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel(metric_name)\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "# This utility function is from the sklearn docs:\n",
    "# http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
    "\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "\n",
    "def predict(model, X_test, y_test):\n",
    "    \"\"\"\n",
    "    This function performs predictions.\n",
    "    \"\"\"\n",
    "    y_softmax = model.predict(X_test)\n",
    "    y_test_1d = []\n",
    "    y_pred_1d = []\n",
    "\n",
    "    for i in range(len(y_test)):\n",
    "        probs = y_test[i]\n",
    "        index_arr = np.nonzero(probs)\n",
    "        one_hot_index = index_arr[0].item(0)\n",
    "        y_test_1d.append(one_hot_index)\n",
    "\n",
    "    for i in range(0, len(y_softmax)):\n",
    "        probs = y_softmax[i]\n",
    "        predicted_index = np.argmax(probs)\n",
    "        y_pred_1d.append(predicted_index)\n",
    "    return y_test_1d, y_pred_1d\n",
    "\n",
    "\n",
    "def test_model(\n",
    "        model,\n",
    "        X_train,\n",
    "        y_train,\n",
    "        X_test,\n",
    "        y_test,\n",
    "        epoch_stop=20,\n",
    "        verbose=0):\n",
    "    \"\"\"\n",
    "    This function trains the model using training, then\n",
    "    tests the model and evaluates using test set for accuracy\n",
    "    and precision.\n",
    "    \"\"\"\n",
    "    start = time.time()\n",
    "    estopping = EarlyStopping(monitor='acc', patience=10)\n",
    "    model.fit(\n",
    "        X_train,\n",
    "        y_train,\n",
    "        epochs=epoch_stop,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        verbose=verbose,\n",
    "        callbacks=[estopping])\n",
    "    results = model.evaluate(X_test, y_test)\n",
    "    end = time.time()\n",
    "    y_test_1d, y_pred_1d = predict(model, X_test, y_test)\n",
    "    cnf_matrix = confusion_matrix(y_test_1d, y_pred_1d)\n",
    "    plt.figure(figsize=(6, 5))\n",
    "    plot_confusion_matrix(\n",
    "        cnf_matrix,\n",
    "        classes=TEXT_LABELS,\n",
    "        title=\"Confusion matrix\")\n",
    "    plt.show()\n",
    "    total_time = end - start\n",
    "    print(\"Model Training and Testing time : %s secs\" % (total_time))\n",
    "    print(\"Model Testing Accuracy : %s \" % (results[1]))\n",
    "    average_precision = average_precision_score(\n",
    "        y_test_1d, y_pred_1d, average='weighted')\n",
    "    print('Average precision score: {0:0.2f}'.format(\n",
    "        average_precision))\n",
    "    return results\n",
    "\n",
    "\n",
    "def test_model_runs(\n",
    "        model,\n",
    "        X_train,\n",
    "        y_train,\n",
    "        X_test,\n",
    "        y_test,\n",
    "        epoch_stop=20,\n",
    "        num_runs=3,\n",
    "        verbose=0):\n",
    "    \"\"\"\n",
    "    This function trains the model and tests it multiple times\n",
    "    to get average metrics.\n",
    "    \"\"\"\n",
    "    sum_result = 0\n",
    "    sq_sum_result = 0\n",
    "    for val in range(0, num_runs):\n",
    "        base_results = test_model(\n",
    "            model,\n",
    "            X_train,\n",
    "            y_train,\n",
    "            X_test,\n",
    "            y_test,\n",
    "            epoch_stop,\n",
    "            verbose)\n",
    "        sum_result = sum_result + base_results[1]\n",
    "        sq_sum_result = sq_sum_result + base_results[1] * base_results[1]\n",
    "    mean_accuracy = sum_result / num_runs\n",
    "    variance = sq_sum_result / num_runs - mean_accuracy * mean_accuracy\n",
    "    stddev = variance**(.5)\n",
    "    return mean_accuracy, stddev\n",
    "\n",
    "\n",
    "def compare_loss(history, base_history, model_name, base_history_name):\n",
    "    loss_base_model = base_history.history['val_loss']\n",
    "    loss_model = history.history['val_loss']\n",
    "\n",
    "    e = range(1, NB_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, loss_base_model, 'bo', label=base_history_name)\n",
    "    plt.plot(e, loss_model, 'red', label=model_name)\n",
    "    plt.legend()\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('loss')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def read_embedding(path):\n",
    "\n",
    "    embeddings_index = {}\n",
    "    f = open(path)\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "\n",
    "    return embeddings_index\n",
    "\n",
    "\n",
    "def create_embedding_matrix(tokenizer, embedding_dim, embeddings_index):\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "    word_embedding_matrix = np.zeros((NB_WORDS + 1, embedding_dim))\n",
    "    for w, i in tk.word_index.items():\n",
    "        # The word_index contains a token for all words of the training data so\n",
    "        # we need to limit that\n",
    "        if i < NB_WORDS:\n",
    "            vect = embeddings_index.get(w)\n",
    "            # Check if the word from the training data occurs in the GloVe word embeddings\n",
    "            # Otherwise the vector is kept with only zeros\n",
    "            if vect is not None:\n",
    "                word_embedding_matrix[i] = vect\n",
    "        else:\n",
    "            break\n",
    "    print('Shape of embedding matrix:', word_embedding_matrix.shape)\n",
    "    return word_embedding_matrix\n",
    "\n",
    "\n",
    "def xml_to_reviews(neg_path, pos_path):\n",
    "    # total number of reviews\n",
    "    reviews = []\n",
    "    negative_reviews = []\n",
    "    positive_reviews = []\n",
    "\n",
    "    neg_tree = ET.parse(neg_path)\n",
    "    neg_root = neg_tree.getroot()\n",
    "    for rev in neg_root.iter('review'):\n",
    "        reviews.append(rev.text)\n",
    "        negative_reviews.append(rev.text)\n",
    "\n",
    "    pos_tree = ET.parse(pos_path)\n",
    "    pos_root = pos_tree.getroot()\n",
    "\n",
    "    for rev in pos_root.iter('review'):\n",
    "        reviews.append(rev.text)\n",
    "        positive_reviews.append(rev.text)\n",
    "\n",
    "    return reviews, negative_reviews, positive_reviews\n",
    "\n",
    "\n",
    "def load_data(domain):\n",
    "    reviews, n, p = xml_to_reviews(\"./data/\" +\n",
    "                                   domain +\n",
    "                                   \"/negative.parsed\", \"./data/\" +\n",
    "                                   domain +\n",
    "                                   \"/positive.parsed\")\n",
    "    negatives_df = pd.DataFrame(np.zeros(len(n)))\n",
    "    positives_df = pd.DataFrame(np.ones(len(p)))\n",
    "    # append the two as one single frame\n",
    "    label_dataframe = negatives_df.append(positives_df)\n",
    "    label_dataframe.columns = [LABELS_COLUMN_NAME]\n",
    "    review_dataframe = pd.DataFrame(reviews)\n",
    "    review_dataframe.columns = [REVIEWS_COLUMN_NAME]\n",
    "    return review_dataframe, label_dataframe, p, n\n",
    "\n",
    "# remove stop words with exceptions\n",
    "\n",
    "\n",
    "def remove_stopwords(input_text):\n",
    "    stopwords_list = stopwords.words('english')\n",
    "    # Some words which might indicate a certain sentiment are kept\n",
    "    whitelist = [\"n't\", \"not\", \"no\"]\n",
    "    words = input_text.split()\n",
    "    clean_words = [word for word in words if (\n",
    "        word not in stopwords_list or word in whitelist) and len(word) > 1]\n",
    "    return \" \".join(clean_words)\n",
    "\n",
    "\n",
    "# tokenization with max words defined and filters\n",
    "tk = Tokenizer(num_words=NB_WORDS,\n",
    "               filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "               lower=True,\n",
    "               split=\" \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6IPzO4J5IFDQ"
   },
   "source": [
    "## Source Reviews loading and Transformations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W9mYomzbIFDQ"
   },
   "outputs": [],
   "source": [
    "# load all the reviews and labels for the source\n",
    "source_review_dataframe, source_label_dataframe, source_positive_dataframe, source_negative_dataframe = load_data(SOURCE_DOMAIN)\n",
    "# remove stop words and do basic preprocessing\n",
    "source_review_dataframe[REVIEWS_COLUMN_NAME] = source_review_dataframe[REVIEWS_COLUMN_NAME].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3Wq-FN9QIFDS",
    "outputId": "45a504a4-c305-4303-f477-5e6ad200c60b"
   },
   "outputs": [],
   "source": [
    "source_data_train, source_data_test, source_label_train, source_label_test = train_test_split(source_review_dataframe[REVIEWS_COLUMN_NAME], source_label_dataframe, test_size=0.2, random_state=37)\n",
    "print('# Target Train data samples:', source_data_train.shape[0])\n",
    "print('# Target Test data samples:', source_data_test.shape[0])\n",
    "assert source_data_train.shape[0] == source_label_train.shape[0]\n",
    "assert source_data_test.shape[0] == source_label_test.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6oCtERfUIFDV"
   },
   "source": [
    "## Target Reviews loading and Transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xWJ7MCaoIFDW"
   },
   "outputs": [],
   "source": [
    "# load all the reviews and labels for the source\n",
    "target_review_dataframe, target_label_dataframe, target_positive_dataframe, target_negative_dataframe = load_data(TARGET_DOMAIN)\n",
    "# remove stop words and do basic preprocessing\n",
    "target_review_dataframe[REVIEWS_COLUMN_NAME] = target_review_dataframe[REVIEWS_COLUMN_NAME].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NB5kNa8mIFDY",
    "outputId": "f406e419-0b67-48a5-928b-2746325426d3"
   },
   "outputs": [],
   "source": [
    "target_data_train, target_data_test, target_label_train, target_label_test = train_test_split(target_review_dataframe[REVIEWS_COLUMN_NAME], target_label_dataframe, test_size=0.2, random_state=37)\n",
    "print('# Target Train data samples:', target_data_train.shape[0])\n",
    "print('# Target Test data samples:', target_data_test.shape[0])\n",
    "assert target_data_train.shape[0] == target_label_train.shape[0]\n",
    "assert target_data_test.shape[0] == target_label_test.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "guAQo_mGIFDa"
   },
   "source": [
    "## Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sufwNDwrIFDb"
   },
   "source": [
    "## 1. Word Distribution in Source and Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9Lkh9yQzIFDc",
    "outputId": "441d1382-f4aa-40dc-8538-9505f01ee737"
   },
   "outputs": [],
   "source": [
    "# WORDS DERIVED COLUMN\n",
    "NUMBER_OF_WORDS_COLUMN = REVIEWS_COLUMN_NAME + 'NB_WORDS_COLUM'\n",
    "# source data\n",
    "source_review_dataframe[NUMBER_OF_WORDS_COLUMN] = source_review_dataframe[REVIEWS_COLUMN_NAME].str.split(\n",
    ").apply(len)\n",
    "source_data = source_review_dataframe[NUMBER_OF_WORDS_COLUMN].describe()\n",
    "\n",
    "quartiles = ['Q1', 'Q2', 'Q3']\n",
    "source = [source_data['25%'], source_data['50%'], source_data['75%']]\n",
    "\n",
    "\n",
    "# target data\n",
    "target_review_dataframe[NUMBER_OF_WORDS_COLUMN] = target_review_dataframe[REVIEWS_COLUMN_NAME].str.split(\n",
    ").apply(len)\n",
    "target_data = target_review_dataframe[NUMBER_OF_WORDS_COLUMN].describe()\n",
    "\n",
    "quartiles = ['Q1', 'Q2', 'Q3']\n",
    "target = [target_data['25%'], target_data['50%'], target_data['75%']]\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "indices = np.arange(3)\n",
    "width = 0.45       # the width of the bars\n",
    "rects1 = ax.bar(indices, source, width, color='tomato')\n",
    "rects2 = ax.bar(indices + width, target, width, color='darkolivegreen')\n",
    "# add some text for labels, title and axes ticks\n",
    "ax.set_ylabel('Number of Words in Text')\n",
    "ax.set_title('Quartile')\n",
    "ax.set_xticks(indices + width / 2)\n",
    "ax.set_xticklabels(('Q1', 'Q2', 'Q3'))\n",
    "ax.legend((rects1[0], rects2[0]), ('Source (Books)', 'Target(Kitchen)'))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "T2bqFwWLIFDe"
   },
   "source": [
    "## 2. Word Cloud Distribution for Source and Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_-GkGrC6IFDf",
    "outputId": "d5e6cab0-6cb6-4300-8c19-e41f52c29a14"
   },
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud, ImageColorGenerator, STOPWORDS\n",
    "words_source_negative = ' '.join(source_negative_dataframe)\n",
    "words_target_negative = ' '.join(target_negative_dataframe)\n",
    "words_source_positive = ' '.join(source_positive_dataframe)\n",
    "words_target_positive = ' '.join(target_positive_dataframe)\n",
    "wordcloud_source_negative = WordCloud(max_words=100, stopwords=STOPWORDS,\n",
    "                                      background_color='black',\n",
    "                                      width=3000,\n",
    "                                      height=2500\n",
    "                                      ).generate(words_source_negative)\n",
    "wordcloud_target_negative = WordCloud(max_words=100, stopwords=STOPWORDS,\n",
    "                                      background_color='black',\n",
    "                                      width=3000,\n",
    "                                      height=2500\n",
    "                                      ).generate(words_target_negative)\n",
    "wordcloud_source_positive = WordCloud(max_words=100, stopwords=STOPWORDS,\n",
    "                                      background_color='black',\n",
    "                                      width=3000,\n",
    "                                      height=2500\n",
    "                                      ).generate(words_source_positive)\n",
    "wordcloud_target_positive = WordCloud(max_words=100, stopwords=STOPWORDS,\n",
    "                                      background_color='black',\n",
    "                                      width=3000,\n",
    "                                      height=2500\n",
    "                                      ).generate(words_target_positive)\n",
    "plt.figure(1, figsize=(8, 8))\n",
    "plt.imshow(wordcloud_source_negative)\n",
    "plt.axis('off')\n",
    "plt.figure(2, figsize=(8, 8))\n",
    "plt.imshow(wordcloud_target_negative)\n",
    "plt.axis('off')\n",
    "plt.figure(3, figsize=(8, 8))\n",
    "plt.imshow(wordcloud_source_positive)\n",
    "plt.axis('off')\n",
    "plt.figure(4, figsize=(8, 8))\n",
    "plt.imshow(wordcloud_target_positive)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LxnyNl8wIFDi",
    "outputId": "9164c4cc-8017-412d-c7f5-37b99a940c16"
   },
   "outputs": [],
   "source": [
    "# tokenize all the reviews to get words from source and target\n",
    "total_corpus = source_review_dataframe[REVIEWS_COLUMN_NAME].append(\n",
    "    target_review_dataframe[REVIEWS_COLUMN_NAME])\n",
    "tk.fit_on_texts(total_corpus.values.tolist())\n",
    "\n",
    "# convert the text to sequence using the source and target  tokenizer\n",
    "source_data_train_seq = tk.texts_to_sequences(\n",
    "    source_data_train.values.tolist())\n",
    "source_data_test_seq = tk.texts_to_sequences(source_data_test.values.tolist())\n",
    "\n",
    "# convert the text to sequence using the source and target  tokenizer\n",
    "target_data_train_seq = tk.texts_to_sequences(\n",
    "    target_data_train.values.tolist())\n",
    "target_data_test_seq = tk.texts_to_sequences(target_data_test.values.tolist())\n",
    "\n",
    "# pad the sequence with maximum length of source and target\n",
    "X_target_full_train = pad_sequences(target_data_train_seq, maxlen=MAX_LEN)\n",
    "X_target_test = pad_sequences(target_data_test_seq, maxlen=MAX_LEN)\n",
    "\n",
    "# pad the sequence with maximum length of source and target\n",
    "X_source_full_train = pad_sequences(source_data_train_seq, maxlen=MAX_LEN)\n",
    "X_source_test = pad_sequences(source_data_test_seq, maxlen=MAX_LEN)\n",
    "\n",
    "# perform encoding of labels\n",
    "le = LabelEncoder()\n",
    "y_source_train_le = le.fit_transform(source_label_train)\n",
    "y_source_test_le = le.transform(source_label_test)\n",
    "y_source_full_train = to_categorical(y_source_train_le)\n",
    "y_source_test = to_categorical(y_source_test_le)\n",
    "\n",
    "y_target_train_le = le.fit_transform(target_label_train)\n",
    "y_target_test_le = le.transform(target_label_test)\n",
    "y_target_full_train = to_categorical(y_target_train_le)\n",
    "y_target_test = to_categorical(y_target_test_le)\n",
    "\n",
    "# labels from encoder mapping\n",
    "TARGET_TEXT_LABELS = le.classes_\n",
    "TEXT_LABELS = le.classes_\n",
    "\n",
    "assert X_source_full_train.shape[0] == y_source_full_train.shape[0]\n",
    "assert X_target_full_train.shape[0] == y_target_full_train.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lDsvIHytIFDj"
   },
   "source": [
    "## Experiment 0: Kim's CNN with no pre-trained Embeddings, train on Source and Test on Target (worst case base to compare and contrast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "k8XbY4JvIFDl",
    "outputId": "3ce0c589-f40e-432e-8bef-cd19e53ee4be"
   },
   "outputs": [],
   "source": [
    "def kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               GLOVE_DIM,\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu')(pool1)\n",
    "    out = Dense(2, activation='softmax')(dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "source_model = kim_cnn()\n",
    "kim_cnn_model_history = train_model(\n",
    "    source_model,\n",
    "    X_source_full_train,\n",
    "    y_source_full_train,\n",
    "    number_of_epochs=15)\n",
    "# test on the source first\n",
    "results = source_model.evaluate(X_source_test, y_source_test)\n",
    "print(\"Model Testing Accuracy Source: %s \" % (results[1]))\n",
    "results = source_model.evaluate(X_target_test, y_target_test)\n",
    "print(\"Model Testing Accuracy Target: %s \" % (results[1]))\n",
    "y_test, y_pred = predict(source_model, X_target_test, y_target_test)\n",
    "cnf_matrix = confusion_matrix(y_test, y_pred)\n",
    "plt.figure(figsize=(6, 5))\n",
    "plot_confusion_matrix(\n",
    "    cnf_matrix,\n",
    "    classes=TEXT_LABELS,\n",
    "    title=\"Confusion matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3z8G1XwRIFDo"
   },
   "outputs": [],
   "source": [
    "from keras.utils import plot_model\n",
    "plot_model(source_model, to_file='model.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VfNANJJ7IFDs"
   },
   "source": [
    "## Experiment 1: Kim's CNN with no pre-trained Embeddings, train on Target and Test on Target (best case base to compare and contrast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FHtEBJjIIFDs",
    "outputId": "5d88af15-f8e5-4968-ca05-ff6b186bb150"
   },
   "outputs": [],
   "source": [
    "def kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               GLOVE_DIM,\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu')(pool1)\n",
    "    out = Dense(2, activation='softmax')(dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "kim_cnn_model = kim_cnn()\n",
    "kim_cnn_model_history = train_model(\n",
    "    kim_cnn_model,\n",
    "    X_target_full_train,\n",
    "    y_target_full_train,\n",
    "    number_of_epochs=15)\n",
    "# run with regularizer afor 3 runs to see performance\n",
    "multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev = test_model_runs(\n",
    "    kim_cnn_model, X_target_full_train, y_target_full_train, X_target_test, y_target_test, 15, 1)\n",
    "print(\n",
    "    \"Mean Test accuracy for Kim's CNN Model on Target Domain is %s and Std Dev %s\" %\n",
    "    (multichannel_multifilter_cnn_model_mean_accuracy,\n",
    "     multichannel_multifilter_cnn_model_std_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OzCqqItpIFD0"
   },
   "source": [
    "## Experiment 2 : Kim's CNN with GloVe Embeddings (frozen) trained on target data and test on unseen target data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vxRL5lKHIFD1",
    "outputId": "96f8d4af-ab28-4962-ff81-caa3cf3d6f08"
   },
   "outputs": [],
   "source": [
    "# read the embedding matrix\n",
    "embeddings_index = read_embedding('./embeddings/glove.6B.100d.txt')\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings_index)\n",
    "\n",
    "\n",
    "def kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               GLOVE_DIM,\n",
    "                               weights=[emb_matrix],\n",
    "                               trainable=False,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu')(pool1)\n",
    "    out = Dense(2, activation='softmax')(dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "kim_cnn_model = kim_cnn()\n",
    "kim_cnn_model_history = train_model(\n",
    "    kim_cnn_model,\n",
    "    X_target_full_train,\n",
    "    y_target_full_train,\n",
    "    number_of_epochs=15)\n",
    "# run with regularizer afor 3 runs to see performance\n",
    "multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev = test_model_runs(\n",
    "    kim_cnn_model, X_target_full_train, y_target_full_train, X_target_test, y_target_test, 15, 1)\n",
    "print(\n",
    "    \"Mean Test accuracy for Kim's CNN Model on Target Domain is %s and Std Dev %s\" %\n",
    "    (multichannel_multifilter_cnn_model_mean_accuracy,\n",
    "     multichannel_multifilter_cnn_model_std_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dxjDxFV3IFD3"
   },
   "source": [
    "## Experiment 3: Kim's CNN with GloVe Embeddings (non-frozen) trained on target data and test on unseen target data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TuVYSpjlIFD4",
    "outputId": "8dc2f74f-410e-4e1f-8c67-d2878f582e33"
   },
   "outputs": [],
   "source": [
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings_index)\n",
    "\n",
    "\n",
    "def kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               GLOVE_DIM,\n",
    "                               weights=[emb_matrix],\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu')(pool1)\n",
    "    out = Dense(2, activation='softmax')(dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "kim_cnn_model = kim_cnn()\n",
    "kim_cnn_model_history = train_model(\n",
    "    kim_cnn_model,\n",
    "    X_target_full_train,\n",
    "    y_target_full_train,\n",
    "    number_of_epochs=15)\n",
    "# run with regularizer afor 3 runs to see performance\n",
    "multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev = test_model_runs(\n",
    "    kim_cnn_model, X_target_full_train, y_target_full_train, X_target_test, y_target_test, 15, 1)\n",
    "print(\n",
    "    \"Mean Test accuracy for Kim's CNN Model on Target Domain is %s and Std Dev %s\" %\n",
    "    (multichannel_multifilter_cnn_model_mean_accuracy,\n",
    "     multichannel_multifilter_cnn_model_std_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YaZ-5AQaIFD6"
   },
   "source": [
    "## Experiment 4: Train on source domain using GloVe embeddings, clone it, train on target domain and test on target with every layer trainable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6PO04lNFIFD7",
    "outputId": "9aa84a83-a1c9-48de-ea00-c8bac75eb28b",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# read the embedding matrix\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings_index)\n",
    "\n",
    "# create feature layers and classification layers\n",
    "graph_in = Input(shape=(NB_WORDS + 1, 100))\n",
    "\n",
    "convs = []\n",
    "for filter_size in range(3, 5):\n",
    "    x = Convolution1D(\n",
    "        128,\n",
    "        filter_size,\n",
    "        padding='same',\n",
    "        activation='relu')(graph_in)\n",
    "    convs.append(x)\n",
    "\n",
    "graph_out = concatenate(convs, axis=1)\n",
    "graph_out = GlobalMaxPooling1D()(graph_out)\n",
    "graph = Model(graph_in, graph_out)\n",
    "\n",
    "feature_layers = [\n",
    "    Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        100,\n",
    "        weights=[emb_matrix],\n",
    "        trainable=True,\n",
    "        input_length=MAX_LEN),\n",
    "    graph]\n",
    "classification_layers = [Dense(128, activation='relu'),\n",
    "                         Dense(2, activation='softmax')]\n",
    "\n",
    "source_model = Sequential(feature_layers + classification_layers)\n",
    "source_model.summary()\n",
    "model_history = train_model(\n",
    "    source_model,\n",
    "    X_source_full_train,\n",
    "    y_source_full_train,\n",
    "    number_of_epochs=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sE8yZglIIFD9",
    "outputId": "bdd84cea-1561-41a7-ef4d-f1696a2d9692"
   },
   "outputs": [],
   "source": [
    "# clone the source model\n",
    "target_model = models.clone_model(source_model)\n",
    "# train the source model on target\n",
    "domain_adaptation_history = train_model(\n",
    "    target_model,\n",
    "    X_target_full_train,\n",
    "    y_target_full_train,\n",
    "    number_of_epochs=15)\n",
    "# run with regularizer afor 3 runs to see performance\n",
    "multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev = test_model_runs(\n",
    "    target_model, X_target_full_train, y_target_full_train, X_target_test, y_target_test, 20, 1)\n",
    "print(\"Mean Test accuracy for Source Model Trained on Source, trained on Target and tested on Target Domain is %s and Std Dev %s\" %\n",
    "      (multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Dg41tQfIIFD_"
   },
   "source": [
    "## Experiment 5: Train on source domain using GloVe embeddings, clone it, train on target domain and test on target with feature layers frozen from source training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_bMRGX1CIFD_",
    "outputId": "5e56e945-fd1d-4f85-e236-a96794c06260"
   },
   "outputs": [],
   "source": [
    "# read the embedding matrix\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings_index)\n",
    "\n",
    "# create feature layers and classification layers\n",
    "graph_in = Input(shape=(NB_WORDS + 1, 100))\n",
    "\n",
    "convs = []\n",
    "for filter_size in range(3, 5):\n",
    "    x = Convolution1D(\n",
    "        128,\n",
    "        filter_size,\n",
    "        padding='same',\n",
    "        activation='relu')(graph_in)\n",
    "    convs.append(x)\n",
    "\n",
    "graph_out = concatenate(convs, axis=1)\n",
    "graph_out = GlobalMaxPooling1D()(graph_out)\n",
    "graph = Model(graph_in, graph_out)\n",
    "\n",
    "feature_layers = [\n",
    "    Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        100,\n",
    "        weights=[emb_matrix],\n",
    "        trainable=True,\n",
    "        input_length=MAX_LEN),\n",
    "    graph]\n",
    "classification_layers = [Dense(128, activation='relu'),\n",
    "                         Dense(2, activation='softmax')]\n",
    "\n",
    "source_model = Sequential(feature_layers + classification_layers)\n",
    "source_model.summary()\n",
    "model_history = train_model(\n",
    "    source_model,\n",
    "    X_source_full_train,\n",
    "    y_source_full_train,\n",
    "    number_of_epochs=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hcMpQEyxIFEC",
    "outputId": "d4ea9a75-8108-45dc-87a4-ef583658b662"
   },
   "outputs": [],
   "source": [
    "# target is source\n",
    "target_model = source_model\n",
    "\n",
    "# freeze the feature layers\n",
    "for l in feature_layers:\n",
    "    l.trainable = False\n",
    "\n",
    "# train the source model on target\n",
    "domain_adaptation_history = train_model(\n",
    "    target_model,\n",
    "    X_target_full_train,\n",
    "    y_target_full_train,\n",
    "    number_of_epochs=15)\n",
    "# run with regularizer afor 3 runs to see performance\n",
    "multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev = test_model_runs(\n",
    "    target_model, X_target_full_train, y_target_full_train, X_target_test, y_target_test, 20, 1)\n",
    "print(\"Mean Test accuracy for Source Model Trained on Source, trained on Target and tested on Target Domain is %s and Std Dev %s\" %\n",
    "      (multichannel_multifilter_cnn_model_mean_accuracy, multichannel_multifilter_cnn_model_std_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZHuI4oLpIFEE"
   },
   "source": [
    "## Experiment 6: Stacked Autoencoders with DNN trained in unsupervised manner on source, feature layers extracted, connected with classification layers to train and test on target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "waxvY45dIFEF",
    "outputId": "5fe73871-ec31-4d3a-f9b8-86491bc330e9"
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(NB_WORDS + 1, 300))\n",
    "model.compile('rmsprop', 'mse')\n",
    "\n",
    "input_i = Input(shape=(300, 300))\n",
    "encoded_h1 = Dense(128, activation='tanh')(input_i)\n",
    "encoded_h2 = Dense(64, activation='tanh')(encoded_h1)\n",
    "encoded_h3 = Dense(32, activation='tanh')(encoded_h2)\n",
    "encoded_h4 = Dense(16, activation='tanh')(encoded_h3)\n",
    "encoded_h5 = Dense(8, activation='tanh')(encoded_h4)\n",
    "latent = Dense(2, activation='tanh')(encoded_h5)\n",
    "decoder_h1 = Dense(8, activation='tanh')(latent)\n",
    "decoder_h2 = Dense(16, activation='tanh')(decoder_h1)\n",
    "decoder_h3 = Dense(32, activation='tanh')(decoder_h2)\n",
    "decoder_h4 = Dense(64, activation='tanh')(decoder_h3)\n",
    "decoder_h5 = Dense(128, activation='tanh')(decoder_h4)\n",
    "\n",
    "output = Dense(300, activation='tanh')(decoder_h5)\n",
    "\n",
    "autoencoder = Model(input_i, output)\n",
    "autoencoder.summary()\n",
    "autoencoder.compile('adadelta', 'mse')\n",
    "\n",
    "X_source_full_train_embedded = model.predict(X_source_full_train)\n",
    "autoencoder.fit(\n",
    "    X_source_full_train_embedded,\n",
    "    X_source_full_train_embedded,\n",
    "    epochs=15,\n",
    "    batch_size=256,\n",
    "    validation_split=.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "myeuzcYeIFEH",
    "outputId": "12536c33-8ed9-4be3-c7f3-3abf07fbe5f0"
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(NB_WORDS + 1, 300))\n",
    "model.compile('rmsprop', 'mse')\n",
    "\n",
    "# train target on same embeddings\n",
    "X_target_full_train_embedded = model.predict(X_target_full_train)\n",
    "\n",
    "classification_model = Sequential()\n",
    "classification_model.add(autoencoder.layers[0])\n",
    "classification_model.add(autoencoder.layers[1])\n",
    "classification_model.add(autoencoder.layers[2])\n",
    "classification_model.add(autoencoder.layers[3])\n",
    "classification_model.add(autoencoder.layers[4])\n",
    "classification_model.add(autoencoder.layers[5])\n",
    "classification_model.add(Flatten())\n",
    "classification_model.add(Dense(2, activation='softmax'))\n",
    "classification_model.compile(optimizer='rmsprop',\n",
    "                             loss='categorical_crossentropy',\n",
    "                             metrics=['accuracy'])\n",
    "classification_model.summary()\n",
    "classification_model.compile(\n",
    "    optimizer=Adam(),\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'])\n",
    "\n",
    "# Train the model, iterating on the data in batches of 32 samples\n",
    "classification_model.fit(\n",
    "    X_target_full_train_embedded,\n",
    "    y_target_full_train,\n",
    "    epochs=20,\n",
    "    batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kh-NDu9RIFEJ",
    "outputId": "96822b59-269c-4df0-cfec-01ef2ff71c5d"
   },
   "outputs": [],
   "source": [
    "# pass the test through embeddings\n",
    "X_target_test_embedded = model.predict(X_target_test)\n",
    "\n",
    "# test the models\n",
    "results = classification_model.evaluate(X_target_test_embedded, y_target_test)\n",
    "end = time.time()\n",
    "y_softmax = classification_model.predict(X_target_test_embedded)\n",
    "y_test_1d = []\n",
    "y_pred_1d = []\n",
    "\n",
    "for i in range(len(y_target_test)):\n",
    "    probs = y_target_test[i]\n",
    "    index_arr = np.nonzero(probs)\n",
    "    one_hot_index = index_arr[0].item(0)\n",
    "    y_test_1d.append(one_hot_index)\n",
    "\n",
    "for i in range(0, len(y_softmax)):\n",
    "    probs = y_softmax[i]\n",
    "    predicted_index = np.argmax(probs)\n",
    "    y_pred_1d.append(predicted_index)\n",
    "cnf_matrix = confusion_matrix(y_test_1d, y_pred_1d)\n",
    "plt.figure(figsize=(6, 5))\n",
    "plot_confusion_matrix(\n",
    "    cnf_matrix,\n",
    "    classes=TEXT_LABELS,\n",
    "    title=\"Confusion matrix\")\n",
    "plt.show()\n",
    "print(\"Model Testing Accuracy : %s \" % (results[1]))\n",
    "average_precision = average_precision_score(\n",
    "    y_target_test, y_softmax, average='weighted')\n",
    "print('Average precision score: {0:0.2f}'.format(\n",
    "      average_precision))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XXM1i3mVIFEM"
   },
   "source": [
    "## Experiment 7: Stacked Autoencoders with CNN and Max Pooling, trained in unsupervised manner on source, feature layers extracted, connected with classification layers to train and test on target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z_XLcS23IFEM",
    "outputId": "19f33ed2-936f-4d5a-987f-b5668ed4f9f5"
   },
   "outputs": [],
   "source": [
    "NUM_WORDS = 3\n",
    "pool_size = 2\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(NB_WORDS + 1, 300))\n",
    "model.compile('rmsprop', 'mse')\n",
    "\n",
    "x = Input(shape=(300, 300))\n",
    "\n",
    "h = x\n",
    "h = Convolution1D(filters=300, kernel_size=NUM_WORDS,\n",
    "                  activation=\"relu\", padding='same', name='Conv1')(h)\n",
    "h = MaxPooling1D(pool_size=pool_size, name='Maxpool1')(h)\n",
    "h = Convolution1D(filters=150, kernel_size=NUM_WORDS,\n",
    "                  activation=\"relu\", padding='same', name='Conv2')(h)\n",
    "h = MaxPooling1D(pool_size=pool_size, name=\"Maxpool2\")(h)\n",
    "h = Flatten()(h)\n",
    "h = Dense(10, name='latent')(h)\n",
    "y = h\n",
    "y = Dense(11250, activation=\"relu\")(y)\n",
    "y = Reshape((75, 150))(y)\n",
    "y = Convolution1D(filters=150, kernel_size=NUM_WORDS,\n",
    "                  activation=\"relu\", padding='same', name='conv-decode1')(y)\n",
    "y = UpSampling1D(size=pool_size, name='upsampling1')(y)\n",
    "y = Convolution1D(filters=300, kernel_size=NUM_WORDS,\n",
    "                  activation=\"relu\", padding='same', name='conv-decode2')(y)\n",
    "y = UpSampling1D(size=pool_size, name='upsampling2')(y)\n",
    "\n",
    "autoencoder = Model(x, y)\n",
    "autoencoder.summary()\n",
    "autoencoder.compile('adadelta', 'mse')\n",
    "\n",
    "\n",
    "X_source_full_train_embedded = model.predict(X_source_full_train)\n",
    "autoencoder.fit(\n",
    "    X_source_full_train_embedded,\n",
    "    X_source_full_train_embedded,\n",
    "    epochs=15,\n",
    "    batch_size=256,\n",
    "    validation_split=.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LxMG__eJIFEP",
    "outputId": "9980bff2-97e3-48f0-da87-98277c04c052",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(NB_WORDS + 1, 300))\n",
    "model.compile('rmsprop', 'mse')\n",
    "\n",
    "# train target on same embeddings\n",
    "X_target_full_train_embedded = model.predict(X_target_full_train)\n",
    "\n",
    "classification_model = Sequential()\n",
    "classification_model.add(autoencoder.layers[0])\n",
    "classification_model.add(autoencoder.layers[1])\n",
    "classification_model.add(autoencoder.layers[2])\n",
    "classification_model.add(autoencoder.layers[3])\n",
    "classification_model.add(autoencoder.layers[4])\n",
    "classification_model.add(autoencoder.layers[5])\n",
    "classification_model.add(Dense(2, activation='softmax'))\n",
    "classification_model.compile(optimizer='rmsprop',\n",
    "                             loss='categorical_crossentropy',\n",
    "                             metrics=['accuracy'])\n",
    "classification_model.summary()\n",
    "classification_model.compile(\n",
    "    optimizer=Adam(),\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'])\n",
    "\n",
    "# Train the model, iterating on the data in batches of 32 samples\n",
    "classification_model.fit(\n",
    "    X_target_full_train_embedded,\n",
    "    y_target_full_train,\n",
    "    epochs=20,\n",
    "    batch_size=32)\n",
    "\n",
    "# pass the test through embeddings\n",
    "X_target_test_embedded = model.predict(X_target_test)\n",
    "\n",
    "# test the models\n",
    "results = classification_model.evaluate(X_target_test_embedded, y_target_test)\n",
    "end = time.time()\n",
    "y_softmax = classification_model.predict(X_target_test_embedded)\n",
    "y_test_1d = []\n",
    "y_pred_1d = []\n",
    "\n",
    "for i in range(len(y_target_test)):\n",
    "    probs = y_target_test[i]\n",
    "    index_arr = np.nonzero(probs)\n",
    "    one_hot_index = index_arr[0].item(0)\n",
    "    y_test_1d.append(one_hot_index)\n",
    "\n",
    "for i in range(0, len(y_softmax)):\n",
    "    probs = y_softmax[i]\n",
    "    predicted_index = np.argmax(probs)\n",
    "    y_pred_1d.append(predicted_index)\n",
    "cnf_matrix = confusion_matrix(y_test_1d, y_pred_1d)\n",
    "plt.figure(figsize=(6, 5))\n",
    "plot_confusion_matrix(\n",
    "    cnf_matrix,\n",
    "    classes=TEXT_LABELS,\n",
    "    title=\"Confusion matrix\")\n",
    "plt.show()\n",
    "print(\"Model Testing Accuracy : %s \" % (results[1]))\n",
    "average_precision = average_precision_score(\n",
    "    y_target_test, y_softmax, average='weighted')\n",
    "print('Average precision score: {0:0.2f}'.format(\n",
    "      average_precision))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JefXtZ14IFEQ"
   },
   "source": [
    "## Transformation to BOW Vectors of Source and Target for Experiments "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OyC3zDSZIFER",
    "outputId": "a95398fe-76b8-4143-b78a-a79dfe59f289"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# load all the reviews and labels for the source and target\n",
    "source_review_dataframe, source_label_dataframe, p, n = load_data(\n",
    "    SOURCE_DOMAIN)\n",
    "target_review_dataframe, target_label_dataframe, p, n = load_data(\n",
    "    TARGET_DOMAIN)\n",
    "\n",
    "# remove stop words and do basic preprocessing\n",
    "source_review_dataframe[REVIEWS_COLUMN_NAME] = source_review_dataframe[REVIEWS_COLUMN_NAME].apply(\n",
    "    remove_stopwords)\n",
    "target_review_dataframe[REVIEWS_COLUMN_NAME] = target_review_dataframe[REVIEWS_COLUMN_NAME].apply(\n",
    "    remove_stopwords)\n",
    "total_corpus = source_review_dataframe[REVIEWS_COLUMN_NAME].append(\n",
    "    target_review_dataframe[REVIEWS_COLUMN_NAME])\n",
    "\n",
    "# n-gram on word level\n",
    "bow_transformer = CountVectorizer(\n",
    "    analyzer='word',\n",
    "    max_features=MAX_FEATURES,\n",
    "    ngram_range=(1,2))\n",
    "\n",
    "# remove all stop words from the corpus\n",
    "total_corpus.apply(remove_stopwords)\n",
    "\n",
    "# fit the entire corpus first\n",
    "bow_transformer.fit(total_corpus)\n",
    "\n",
    "# size of the vocabulary\n",
    "print(len(bow_transformer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "knMlNgvLIFET",
    "outputId": "92b1625a-59ba-4528-bfbe-fce410664952"
   },
   "outputs": [],
   "source": [
    "# transform source data to vectors\n",
    "source_review_arrays = bow_transformer.transform(\n",
    "    source_review_dataframe[REVIEWS_COLUMN_NAME]).toarray()\n",
    "print(source_review_arrays.shape)\n",
    "\n",
    "# transform target data to vectors\n",
    "target_review_arrays = bow_transformer.transform(\n",
    "    target_review_dataframe[REVIEWS_COLUMN_NAME]).toarray()\n",
    "\n",
    "# entire source labels as encoded\n",
    "# perform encoding of labels\n",
    "le = LabelEncoder()\n",
    "source_label_encoded = le.fit_transform(source_label_dataframe)\n",
    "target_label_encoded = le.fit_transform(target_label_dataframe)\n",
    "\n",
    "# source splits\n",
    "source_data_train, source_data_test, source_label_train, source_label_test = train_test_split(\n",
    "    source_review_arrays, source_label_encoded, test_size=0.2, random_state=37)\n",
    "\n",
    "# target splits\n",
    "target_data_train, target_data_test, target_label_train, target_label_test = train_test_split(\n",
    "    target_review_arrays, target_label_encoded, test_size=0.2, random_state=37)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mTBF-HfkIFEW"
   },
   "source": [
    "## Experiment 8: Deep CORAL: Correlation Alignment for Deep Domain Adaptation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_j1iKOFbIFEW"
   },
   "source": [
    "By Baochen Sun, Kate Saenko \n",
    "https://arxiv.org/abs/1607.01719\n",
    "\n",
    "Deep neural networks are able to learn powerful representations from large quantities of labeled input data, however they cannot always generalize well across changes in input distributions. Domain adaptation algorithms have been proposed to compensate for the degradation in performance due to domain shift. In this paper, we address the case when the target domain is unlabeled, requiring unsupervised adaptation. CORAL is a \"frustratingly easy\" unsupervised domain adaptation method that aligns the second-order statistics of the source and target distributions with a linear transformation. Here, we extend CORAL to learn a nonlinear transformation that aligns correlations of layer activations in deep neural networks (Deep CORAL). Experiments on standard benchmark datasets show state-of-the-art performance.\n",
    "<img src=\"attachment:DeepCORAL.png\" width=\"500\", height=\"400\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uavQlpZ4IFEW"
   },
   "outputs": [],
   "source": [
    "# moment alignment neural network algorithm\n",
    "from correlation_alignment import CORALRegularizer\n",
    "from central_moment_discrepancy import CMDRegularizer\n",
    "from maximum_mean_discrepancy import MMDRegularizer\n",
    "from mann_sentiment_analysis import MANN\n",
    "\n",
    "MANN_HIDDEN_UNITS = 50\n",
    "MANN_MAX_N_EPOCH = 1500\n",
    "MANN_BATCH_SIZE = 300\n",
    "\n",
    "def accuracy(y, y_true):\n",
    "    \"\"\"\n",
    "    amount of right classified reviews\n",
    "    \"\"\"\n",
    "    return 1 - np.sum(np.abs(np.round(y).ravel() -\n",
    "                             y_true.ravel())) / y.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Gco8jeHTIFEX",
    "outputId": "021945b1-aec8-449f-dd83-4889de493391",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Neural Network fit\n",
    "nn = MANN(n_features=MAX_FEATURES, n_hiddens=MANN_HIDDEN_UNITS,\n",
    "          n_epochs=MANN_MAX_N_EPOCH, bsize=MANN_BATCH_SIZE, save_weights='nn')\n",
    "nn.fit(source_data_train, source_label_train, target_data_train, verbose=1)\n",
    "\n",
    "# deep coral\n",
    "coral = CORALRegularizer(1.)\n",
    "deep_coral = MANN(n_features=MAX_FEATURES, n_hiddens=MANN_HIDDEN_UNITS,\n",
    "                  n_epochs=MANN_MAX_N_EPOCH, bsize=MANN_BATCH_SIZE,\n",
    "                  activity_regularizer=coral)\n",
    "deep_coral.fit(\n",
    "    source_data_train,\n",
    "    source_label_train,\n",
    "    target_data_train,\n",
    "    verbose=1,\n",
    "    init_weights='nn')\n",
    "target_test_predictions = deep_coral.predict(target_data_test)\n",
    "print('Accuracy of deep coral: ' +\n",
    "      str(accuracy(target_label_test, target_test_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "collapsed": true,
    "id": "l4JuVU1jIFEa"
   },
   "source": [
    "## Experiment 9:  Central moment discrepancy (CMD) for domain-invariant representation learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BVZK6BUNIFEa"
   },
   "source": [
    "<img src=\"attachment:CMD-Paper.png\">\n",
    "\n",
    "W.Zellinger, T. Grubinger, E. Lughofer, T. Natschlaeger, and Susanne Saminger-Platz, \"Central moment discrepancy (cmd) for domain-invariant representation learning,\" International Conference on Learning Representations (ICLR), 2017 (Abstract)\n",
    "\n",
    "The learning of domain-invariant representations in the context of domain adaptation\n",
    "with neural networks is considered. We propose a new regularization\n",
    "method that minimizes the domain-specific latent feature representations directly\n",
    "in the hidden activation space. Although some standard distribution matching\n",
    "approaches exist that can be interpreted as the matching of weighted sums of\n",
    "moments, e.g. Maximum Mean Discrepancy, an explicit order-wise matching of\n",
    "higher order moments has not been considered before. We propose to match the\n",
    "higher order central moments of probability distributions by means of order-wise\n",
    "moment differences. Our model does not require computationally expensive distance\n",
    "and kernel matrix computations. We utilize the equivalent representation of\n",
    "probability distributions by moment sequences to define a new distance function,\n",
    "called Central Moment Discrepancy (CMD). We prove that CMD is a metric on\n",
    "the set of probability distributions on a compact interval. We further prove that\n",
    "convergence of probability distributions on compact intervals w. r. t. the new metric\n",
    "implies convergence in distribution of the respective random variables. We test\n",
    "our approach on two different benchmark data sets for object recognition (Office)\n",
    "and sentiment analysis of product reviews (Amazon reviews). CMD achieves a\n",
    "new state-of-the-art performance on most domain adaptation tasks of Office and\n",
    "outperforms networks trained with Maximum Mean Discrepancy, Variational Fair\n",
    "Autoencoders and Domain Adversarial Neural Networks on Amazon reviews. In\n",
    "addition, a post-hoc parameter sensitivity analysis shows that the new approach\n",
    "is stable w. r. t. parameter changes in a certain interval. \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FTgisJ6zIFEb"
   },
   "source": [
    "<img src=\"attachment:CMD-Paper.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "u-a95_mRIFEc",
    "outputId": "5b796baa-9f77-4fb1-d7b1-860fcd9ec5ac",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cmd = CMDRegularizer(1.)\n",
    "deep_coral = MANN(n_features=MAX_FEATURES, n_hiddens=MANN_HIDDEN_UNITS,\n",
    "                  n_epochs=MANN_MAX_N_EPOCH, bsize=MANN_BATCH_SIZE,\n",
    "                  activity_regularizer=cmd)\n",
    "deep_coral.fit(\n",
    "    source_data_train,\n",
    "    source_label_train,\n",
    "    target_data_train,\n",
    "    verbose=1,\n",
    "    init_weights='nn')\n",
    "target_test_predictions = deep_coral.predict(target_data_test)\n",
    "print('Accuracy of deep CMD: ' +\n",
    "      str(accuracy(target_label_test, target_test_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NKZ5sbpPIFEg"
   },
   "source": [
    "## Experiment 10: Deep Maximum Mean Discrepancy (MMD)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WuMuiNDXIFEg"
   },
   "source": [
    "https://www.ijcai.org/Proceedings/09/Papers/200.pdf\n",
    "\n",
    "Domain Adaptation via Transfer Component Analysis\n",
    "Sinno Jialin Pan, Ivor W. Tsang, James T. Kwok1 and Qiang Yang\n",
    "\n",
    "(Abstract)\n",
    "Domain adaptation solves a learning problem in a target domain by utilizing the training data in a different but related source domain. Intuitively, discovering a good feature representation across domains is crucial. In this paper, we propose to find such a representation through a new learning method, transfer component analysis (TCA), for domain adaptation. TCA tries to learn some transfer components across domains in a Reproducing Kernel Hilbert Space (RKHS) using Maximum Mean Discrepancy (MMD). In the subspace spanned by these transfer components, data distributions in different domains are close to each other. As a result, with the new representations in this subspace, we can apply standard machine learning methods to train classifiers or regression models in the source domain for use in the target domain. The main contribution of our work is that we propose a novel feature representation in which to perform domain adaptation via a new parametric kernel using feature extraction methods, which can dramatically minimize the distance between domain distributions by projecting data onto the learned transfer components. Furthermore, our approach can handle large datsets and naturally lead to out-of-sample generalization. The effectiveness and efficiency of our approach in are verified by experiments on two real-world applications: cross-domain indoor WiFi localization and cross-domain text classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "joxyYnWSIFEh",
    "outputId": "76d9f04d-fee8-4db7-8166-049500e98d4d",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import pdist\n",
    "\n",
    "# Neural Network fit\n",
    "nn = MANN(n_features=MAX_FEATURES, n_hiddens=MANN_HIDDEN_UNITS,\n",
    "          n_epochs=MANN_MAX_N_EPOCH, bsize=MANN_BATCH_SIZE, save_weights='nn')\n",
    "nn.fit(source_data_train, source_label_train, target_data_train, verbose=1)\n",
    "\n",
    "# use the activations\n",
    "a_s, a_t = nn.get_activations(source_data_train, target_data_train)\n",
    "a = np.concatenate((a_s, a_t), axis=0)\n",
    "b = 1.0 / np.median(pdist(a))\n",
    "\n",
    "# train with mmd\n",
    "mmd = MMDRegularizer(1., beta=b)\n",
    "deep_coral = MANN(n_features=MAX_FEATURES, n_hiddens=MANN_HIDDEN_UNITS,\n",
    "                  n_epochs=MANN_MAX_N_EPOCH, bsize=MANN_BATCH_SIZE,\n",
    "                  activity_regularizer=mmd)\n",
    "deep_coral.fit(\n",
    "    source_data_train,\n",
    "    source_label_train,\n",
    "    target_data_train,\n",
    "    verbose=1,\n",
    "    init_weights='nn')\n",
    "target_test_predictions = deep_coral.predict(target_data_test)\n",
    "print('Accuracy of deep MMD: ' +\n",
    "      str(accuracy(target_label_test, target_test_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BFS3dQcnIFEk"
   },
   "source": [
    "## Experiment 11: Marginalized Stacked Denoising Autoencoders (mSDA) for Domain Adaptation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qjzuhYmWIFEl"
   },
   "source": [
    "Minmin Chen, Zhixiang (Eddie) Xu, Kilian Q. Weinberger\n",
    "\n",
    "(Abstract)\n",
    "Stacked denoising autoencoders (SDAs) have been successfully used to learn new representations for domain adaptation. Recently, they have attained record accuracy on standard benchmark tasks of sentiment analysis across different text domains. SDAs learn robust data representations by reconstruction, recovering original features from data that are artificially corrupted with noise. In this paper, we propose marginalized SDA (mSDA) that addresses two crucial limitations of SDAs: high computational cost and lack of scalability to high-dimensional features. In contrast to SDAs, our approach of mSDA marginalizes noise and thus does not require stochastic gradient descent or other optimization algorithms to learn parameters — in fact, they are computed in closed-form. Consequently, mSDA, which can be implemented in only 20 lines of MATLABTM, significantly speeds up SDAs by two orders of magnitude. Furthermore, the representations learnt by mSDA are as effective as the traditional SDAs, attaining almost identical accuracies in benchmark tasks.\n",
    "<img src=\"attachment:\" width=\"600\", height=\"400\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zZu3OKtpIFEl"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def compute_msda_representation(x_source, x_target, x_test):\n",
    "    nb_souce = np.shape(x_source)[0]\n",
    "    x_train = np.vstack((x_source, x_target))\n",
    "    new_x_train, W = msda_fit(x_train.T)\n",
    "    mew_x_source = new_x_train[:,:nb_souce].T\n",
    "    new_x_target = new_x_train[:,nb_souce:].T\n",
    "    new_x_test = msda_forward(x_test.T, W).T\n",
    "    return mew_x_source, new_x_target, new_x_test\n",
    "\n",
    "\n",
    "def mda_fit(X, noise=0.5, eta=1e-5):\n",
    "    \"\"\"\n",
    "    inputs: \n",
    "        X : d x n input (Transpose of the usual data-matrix)\n",
    "        noise: corruption level\n",
    "        eta: regularization \n",
    "    \n",
    "    outputs:\n",
    "        hx: d x n hidden representation\n",
    "        W: d x (d+1) mapping\n",
    "    \"\"\"\n",
    "    d, n = np.shape(X)\n",
    "    \n",
    "    # adding bias\n",
    "    Xb = np.vstack((X, np.ones(n)))\n",
    "    \n",
    "    # scatter matrix S\n",
    "    S = np.dot(Xb, Xb.T)\n",
    "    \n",
    "    # corruption vector\n",
    "    q = np.ones((d+1, 1)) * (1.-noise)\n",
    "    q[-1] = 1\n",
    "    \n",
    "    # Q: (d+1)x(d+1)\n",
    "    Q = S*np.dot(q,q.T)\n",
    "    Q[np.diag_indices_from(Q)] = q.T[0] * np.diag(S)\n",
    "\n",
    "    #P: dx(d+1)\n",
    "    P = S[0:-1,:] * q.T \n",
    "    \n",
    "    # final W = P*Q^-1, dx(d+1)\n",
    "    reg = eta * np.eye(d+1)\n",
    "    reg[-1,-1] = 0\n",
    "    W = np.linalg.solve(Q.T + reg, P.T).T\n",
    "\n",
    "    hx = np.tanh(np.dot(W, Xb))\n",
    "    return hx, W\n",
    "\n",
    "   \n",
    "def msda_fit(X, noise=0.5, nb_layers=5, verbose=False):\n",
    "    \"\"\"\n",
    "    inputs:\n",
    "        X : d x n input Transpose of the usual data-matrix)\n",
    "        noise: corruption level\n",
    "        nb_layers: number of layers to stack\n",
    "\n",
    "    outputs:\n",
    "        allhx: (1+nb_layers)*d x n stacked hidden representations\n",
    "        W_list: list of mapping (of size nb_layers)\n",
    "    \"\"\"\n",
    "    eta = 1e-05\n",
    "    allhx = X.copy()\n",
    "    prevhx = X\n",
    "    W_list = []\n",
    "    \n",
    "    for i in range(nb_layers):\n",
    "        if verbose: print('layer =', i)\n",
    "        newhx, W = mda_fit(prevhx, noise, eta)\n",
    "        W_list.append(W)\n",
    "        allhx = np.vstack((allhx, newhx))\n",
    "        prevhx = newhx\n",
    "\n",
    "    return allhx, W_list\n",
    "\n",
    "\n",
    "def msda_forward(X, W_list):\n",
    "    \"\"\"\n",
    "    inputs: \n",
    "        X : d x n input (Transpose of the usual data-matrix)\n",
    "        noise: corruption level\n",
    "        W_list: list of mapping (of size nb_layers) learned by mSDA.\n",
    "    \n",
    "    outputs:\n",
    "        allhx: (1+nb_layers)*d x n stacked hidden representations of X.\n",
    "        \n",
    "    \"\"\"\n",
    "    _, n = np.shape(X)   \n",
    "    hx = X    \n",
    "    \n",
    "    allhx = X.copy()\n",
    "    for W in W_list:\n",
    "        hxb = np.vstack(( hx, np.ones(n)) )\n",
    "        hx = np.tanh( np.dot(W, hxb) )        \n",
    "        allhx = np.vstack( (allhx, hx) )\n",
    "        \n",
    "    return allhx        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "geV9TYfJIFEo"
   },
   "outputs": [],
   "source": [
    "xs_msda, xt_msda, xtest_msda = compute_msda_representation(source_data_train, target_data_train, target_data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N9Vxrc76IFEq"
   },
   "outputs": [],
   "source": [
    "# train representation\n",
    "X_train_reps = np.hstack([source_data_train, xs_msda])\n",
    "\n",
    "# test representation\n",
    "X_test_reps = np.hstack([target_data_test, xtest_msda])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TBnVHYsLIFEs",
    "outputId": "e32f54a9-6eee-4211-918d-d981f916f893"
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "classifier = svm.SVC().fit(X_train_reps, source_label_train)\n",
    "prediction_test = classifier.predict(X_test_reps)\n",
    "print('Test Error       = %f' % np.mean(prediction_test != target_label_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "agoZjbw0IFEw"
   },
   "source": [
    "## Experiment 12: Domain-Adversarial Training of Neural Networks\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Adkss9wLIFEw"
   },
   "source": [
    "https://arxiv.org/abs/1505.07818\n",
    "Yaroslav Ganin, Evgeniya Ustinova, Hana Ajakan, Pascal Germain, Hugo Larochelle, François Laviolette, Mario Marchand, Victor Lempitsky.\n",
    "\n",
    "We introduce a new representation learning approach for domain adaptation, in which data at training and test time come from similar but different distributions. Our approach is directly inspired by the theory on domain adaptation suggesting that, for effective domain transfer to be achieved, predictions must be made based on features that cannot discriminate between the training (source) and test (target) domains. The approach implements this idea in the context of neural network architectures that are trained on labeled data from the source domain and unlabeled data from the target domain (no labeled target-domain data is necessary). As the training progresses, the approach promotes the emergence of features that are (i) discriminative for the main learning task on the source domain and (ii) indiscriminate with respect to the shift between the domains. We show that this adaptation behaviour can be achieved in almost any feed-forward model by augmenting it with few standard layers and a new gradient reversal layer. The resulting augmented architecture can be trained using standard backpropagation and stochastic gradient descent, and can thus be implemented with little effort using any of the deep learning packages. We demonstrate the success of our approach for two distinct classification problems (document sentiment analysis and image classification), where state-of-the-art domain adaptation performance on standard benchmarks is achieved. We also validate the approach for descriptor learning task in the context of person re-identification application.\n",
    "<img src=\"attachment:DANN.png\" width=\"600\", height=\"400\">\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "k19thY50IFEy"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "from math import sqrt\n",
    "\n",
    "# Code adapted from https://github.com/GRAAL-Research/domain_adversarial_neural_network\n",
    "\n",
    "class DANN(object):\n",
    "    \n",
    "    def __init__(self, learning_rate=0.05, hidden_layer_size=25, lambda_adapt=1., maxiter=200,  \n",
    "                 epsilon_init=None, adversarial_representation=True, seed=12342, verbose=False):\n",
    "        \"\"\"\n",
    "        Domain Adversarial Neural Network for classification\n",
    "        \n",
    "        option \"learning_rate\" is the learning rate of the neural network.\n",
    "        option \"hidden_layer_size\" is the hidden layer size.\n",
    "        option \"lambda_adapt\" weights the domain adaptation regularization term.\n",
    "                if 0 or None or False, then no domain adaptation regularization is performed\n",
    "        option \"maxiter\" number of training iterations.\n",
    "        option \"epsilon_init\" is a term used for initialization.\n",
    "                if None the weight matrices are weighted by 6/(sqrt(r+c))\n",
    "                (where r and c are the dimensions of the weight matrix)\n",
    "        option \"adversarial_representation\": if False, the adversarial classifier is trained\n",
    "                but has no impact on the hidden layer representation. The label predictor is\n",
    "                then the same as a standard neural-network one (see experiments_moon.py figures). \n",
    "        option \"seed\" is the seed of the random number generator.\n",
    "        \"\"\"\n",
    "        \n",
    "        self.hidden_layer_size = hidden_layer_size\n",
    "        self.maxiter = maxiter\n",
    "        self.lambda_adapt = lambda_adapt if lambda_adapt not in (None, False) else 0.\n",
    "        self.epsilon_init = epsilon_init\n",
    "        self.learning_rate = learning_rate\n",
    "        self.adversarial_representation = adversarial_representation\n",
    "        self.seed = seed\n",
    "        self.verbose = verbose\n",
    "            \n",
    "    def sigmoid(self, z):\n",
    "        \"\"\"\n",
    "        Sigmoid function.\n",
    "        \n",
    "        \"\"\"\n",
    "        return 1. / (1. + np.exp(-z))\n",
    "    \n",
    "    def softmax(self, z):\n",
    "        \"\"\"\n",
    "        Softmax function.\n",
    "        \n",
    "        \"\"\"\n",
    "        v = np.exp(z)\n",
    "        return v / np.sum(v,axis=0)\n",
    "       \n",
    "    def random_init(self, l_in, l_out):\n",
    "        \"\"\"\n",
    "        This method is used to initialize the weight matrices of the DA neural network \n",
    "        \n",
    "        \"\"\"\n",
    "        if self.epsilon_init is not None:\n",
    "            epsilon = self.epsilon_init \n",
    "        else:\n",
    "            epsilon = sqrt(6.0 / (l_in + l_out))\n",
    "            \n",
    "        return epsilon * (2 * np.random.rand(l_out, l_in) - 1.0)\n",
    "       \n",
    "    def fit(self, X, Y, X_adapt, X_valid=None, Y_valid=None, do_random_init=True):\n",
    "        \"\"\"         \n",
    "        Trains the domain adversarial neural network until it reaches a total number of\n",
    "        iterations of \"self.maxiter\" since it was initialize.\n",
    "        inputs:\n",
    "              X : Source data matrix\n",
    "              Y : Source labels\n",
    "              X_adapt : Target data matrix\n",
    "              (X_valid, Y_valid) : validation set used for early stopping.\n",
    "              do_random_init : A boolean indicating whether to use random initialization or not.\n",
    "        \"\"\"\n",
    "        \n",
    "        nb_examples, nb_features = np.shape(X)\n",
    "        nb_labels = len(set(Y))\n",
    "        nb_examples_adapt, _ = np.shape(X_adapt)\n",
    "\n",
    "        if self.verbose:\n",
    "            print('[DANN parameters]', self.__dict__)\n",
    "        \n",
    "        np.random.seed(self.seed)\n",
    "        \n",
    "        if do_random_init:\n",
    "            W = self.random_init(nb_features, self.hidden_layer_size)\n",
    "            V = self.random_init(self.hidden_layer_size, nb_labels)\n",
    "            b = np.zeros(self.hidden_layer_size)\n",
    "            c = np.zeros(nb_labels)\n",
    "            U = np.zeros(self.hidden_layer_size)\n",
    "            d = 0.\n",
    "        else:\n",
    "            W, V, b, c, U, d = self.W, self.V, self.b, self.c, self.U, self.d \n",
    "            \n",
    "        best_valid_risk = 2.0\n",
    "        continue_until = 30\n",
    "\n",
    "        for t in range(self.maxiter):\n",
    "            for i in range(nb_examples):\n",
    "                x_t, y_t = X[i,:], Y[i]\n",
    "                \n",
    "                hidden_layer = self.sigmoid(np.dot(W, x_t) + b)\n",
    "                output_layer = self.softmax(np.dot(V, hidden_layer) + c)\n",
    "                \n",
    "                y_hot = np.zeros(nb_labels)\n",
    "                y_hot[y_t] = 1.0\n",
    "                 \n",
    "                delta_c = output_layer - y_hot  \n",
    "                delta_V = np.dot(delta_c.reshape(-1,1), hidden_layer.reshape(1,-1)) \n",
    "                delta_b = np.dot(V.T, delta_c) * hidden_layer * (1.-hidden_layer) \n",
    "                delta_W = np.dot(delta_b.reshape(-1,1), x_t.reshape(1,-1)) \n",
    "                \n",
    "                if self.lambda_adapt == 0.:\n",
    "                    delta_U, delta_d = 0., 0.\n",
    "                else:\n",
    "                    # add domain adaptation regularizer from current domain\n",
    "                    gho_x_t = self.sigmoid(np.dot(U.T, hidden_layer) + d)\n",
    "                    \n",
    "                    delta_d = self.lambda_adapt * (1. - gho_x_t) \n",
    "                    delta_U = delta_d * hidden_layer \n",
    "\n",
    "                    if self.adversarial_representation:\n",
    "                        tmp = delta_d * U * hidden_layer * (1. - hidden_layer)\n",
    "                        delta_b += tmp\n",
    "                        delta_W += tmp.reshape(-1,1) * x_t.reshape(1,-1)\n",
    "                    \n",
    "                    # add domain adaptation regularizer from other domain\n",
    "                    i_2 = np.random.randint(nb_examples_adapt)\n",
    "                    x_t_2 = X_adapt[i_2, :]\n",
    "                    hidden_layer_2 = self.sigmoid( np.dot(W, x_t_2) + b)\n",
    "                    gho_x_t_2 = self.sigmoid(np.dot(U.T, hidden_layer_2) + d) \n",
    "                    \n",
    "                    delta_d -= self.lambda_adapt * gho_x_t_2 \n",
    "                    delta_U -= self.lambda_adapt * gho_x_t_2 * hidden_layer_2\n",
    "\n",
    "                    if self.adversarial_representation:\n",
    "                        tmp = -self.lambda_adapt * gho_x_t_2 * U * hidden_layer_2 * (1. - hidden_layer_2)\n",
    "                        delta_b += tmp\n",
    "                        delta_W += tmp.reshape(-1,1) * x_t_2.reshape(1,-1)\n",
    "          \n",
    "                W -= delta_W * self.learning_rate\n",
    "                b -= delta_b * self.learning_rate\n",
    "     \n",
    "                V -= delta_V * self.learning_rate\n",
    "                c -= delta_c * self.learning_rate\n",
    "                \n",
    "                U += delta_U * self.learning_rate \n",
    "                d += delta_d * self.learning_rate \n",
    "            # END for i in range(nb_examples)\n",
    "\n",
    "            self.W, self.V, self.b, self.c, self.U, self.d = W, V, b, c, U, d\n",
    "            \n",
    "            # early stopping\n",
    "            if X_valid is not None:\n",
    "                valid_pred = self.predict(X_valid)\n",
    "                valid_risk = np.mean( valid_pred != Y_valid )\n",
    "                if valid_risk <= best_valid_risk:\n",
    "                    if self.verbose: \n",
    "                        print('[DANN best valid risk so far] %f (iter %d)' % (valid_risk, t))\n",
    "                    best_valid_risk = valid_risk\n",
    "                    best_weights = (W.copy(), V.copy(), b.copy(), c.copy())\n",
    "                    best_t = t\n",
    "                    continue_until = max(continue_until, int(1.5*t))\n",
    "                elif t > continue_until: \n",
    "                    if self.verbose: \n",
    "                        print('[DANN early stop] iter %d' % t)\n",
    "                    break\n",
    "        # END for t in range(self.maxiter)\n",
    "        \n",
    "        if X_valid is not None:\n",
    "            self.W, self.V, self.b, self.c = best_weights\n",
    "            self.nb_iter = best_t\n",
    "            self.valid_risk = best_valid_risk\n",
    "        else:\n",
    "            self.nb_iter = self.maxiter\n",
    "            self.valid_risk = 2.\n",
    "            \n",
    "    def forward(self, X):\n",
    "        \"\"\"\n",
    "         Compute and return the network outputs for X, i.e., a 2D array of size len(X) by len(set(Y)).\n",
    "         the ith row of the array contains output probabilities for each class for the ith example.\n",
    "         \n",
    "        \"\"\"\n",
    "        hidden_layer = self.sigmoid(np.dot(self.W, X.T) + self.b[:,np.newaxis])\n",
    "        output_layer = self.softmax(np.dot(self.V, hidden_layer) + self.c[:,np.newaxis])\n",
    "        return output_layer\n",
    "\n",
    "    def hidden_representation(self, X):\n",
    "        \"\"\"\n",
    "         Compute and return the network hidden layer values for X.\n",
    "        \"\"\"\n",
    "        hidden_layer = self.sigmoid(np.dot(self.W, X.T) + self.b[:,np.newaxis])\n",
    "        return hidden_layer.T\n",
    "\n",
    "    def predict(self, X):\n",
    "        \"\"\"\n",
    "         Compute and return the label predictions for X, i.e., a 1D array of size len(X).\n",
    "         the ith row of the array contains the predicted class for the ith example .\n",
    "        \"\"\"\n",
    "        output_layer = self.forward(X)\n",
    "        return np.argmax(output_layer, 0)\n",
    "\n",
    "    def predict_domain(self, X):\n",
    "        \"\"\"\n",
    "         Compute and return the domain predictions for X, i.e., a 1D array of size len(X).\n",
    "         the ith row of the array contains the predicted domain (0 or 1) for the ith example.\n",
    "        \"\"\"\n",
    "        hidden_layer = self.sigmoid(np.dot(self.W, X.T) + self.b[:, np.newaxis])\n",
    "        output_layer = self.sigmoid(np.dot(self.U, hidden_layer) + self.d)\n",
    "        return np.array(output_layer < .5, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NCZaQ_moIFE0"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from DANN import DANN\n",
    "from sklearn.datasets import load_svmlight_files\n",
    "from sklearn import svm\n",
    "\n",
    "adversarial = True\n",
    "msda = False\n",
    "hidden_layer_size = 50\n",
    "lambda_adapt = 0.1 if adversarial else 0.\n",
    "learning_rate = 0.001 if not msda else 0.0001\n",
    "maxiter = 200\n",
    "\n",
    "dann = DANN(\n",
    "    lambda_adapt=lambda_adapt,\n",
    "    hidden_layer_size=hidden_layer_size,\n",
    "    learning_rate=learning_rate,\n",
    "    maxiter=maxiter,\n",
    "    epsilon_init=None,\n",
    "    seed=12342,\n",
    "    adversarial_representation=adversarial,\n",
    "    verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gTHZNpPWIFE3",
    "outputId": "4552356e-0368-44f7-f90e-f658310f3cbf"
   },
   "outputs": [],
   "source": [
    "dann.fit(\n",
    "    source_data_train,\n",
    "    source_label_train,\n",
    "    target_data_train,\n",
    "    source_data_test,\n",
    "    source_label_test)\n",
    "prediction_test = dann.predict(target_data_test)\n",
    "print('Test Error       = %f' % np.mean(prediction_test != target_label_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UFXLmu1BIFE4"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Chapter 8 Case Study Cross-Domain Sentiment Adaptation-Book-Kitchen.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}