seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Based on the followings:\n",
    "* http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf\n",
    "* http://adventuresinmachinelearning.com/keras-lstm-tutorial/\n",
    "* https://machinelearningmastery.com/configure-encoder-decoder-model-neural-machine-translation/\n",
    "* https://machinelearningmastery.com/develop-encoder-decoder-model-sequence-sequence-prediction-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import seq2seq\n",
    "\n",
    "from string import punctuation\n",
    "from itertools import islice\n",
    "from nltk import corpus, stem\n",
    "from gensim.models import KeyedVectors\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, LSTM, Dense, Masking\n",
    "from keras.preprocessing import sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = 'data'\n",
    "OUTPUT_PATH = 'output'\n",
    "\n",
    "punct = set(punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_embeddings(path, binary=False):\n",
    "    return KeyedVectors.load_word2vec_format(path, binary=binary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_to_int(text, word2idx):\n",
    "    \"\"\"\n",
    "    Convert the provided text to a list of integers, selected according to the provided mapping\n",
    "    @text the text to be converted\n",
    "    @word2idx: a dictionary defining the word->int mapping\n",
    "    \"\"\"\n",
    "    return [word2idx[w] if w in word2idx else 0 for w in re.split(' ')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_contractions(text, w2v):\n",
    "    cont = Contractions(w2v_model=w2v)\n",
    "    return cont.expand_texts(text=text, precise=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    text = text.replace('  ', ' ')\n",
    "    return re.sub(repl='', string=text, pattern='[{}\\n\\t]'.format(''.join(punctuation)))\n",
    "\n",
    "def build_vocabulary(infile, w2v):\n",
    "    idx2word = set() # only count unique words\n",
    "    \n",
    "    with open(infile, 'r', encoding='utf-8') as f:            \n",
    "        for line in f:\n",
    "            line = preprocess(line)\n",
    "            for word in line.split(' '):\n",
    "                if len(word) > 0:\n",
    "                    idx2word.add(word)\n",
    "            \n",
    "    idx2word = list(idx2word)\n",
    "    idx2word.insert(0, '<PAD>')\n",
    "    idx2word.insert(1, '<UNK>')\n",
    "\n",
    "    vocab_size, vocab_dim = len(idx2word), w2v.vector_size\n",
    "    word2idx = {w:i for i,w in enumerate(idx2word)}\n",
    "    word2embeddings = {w:w2v[w] if w in w2v else np.zeros(vocab_dim) for w in idx2word}\n",
    "    \n",
    "    \n",
    "    return idx2word, word2idx, word2embeddings, (vocab_size, vocab_dim)\n",
    "\n",
    "# this assumes no newlines characters\n",
    "def sentence_to_embeddings(sentence, embeddings):\n",
    "    return [embeddings[word] for word in sentence.split(' ') if len(word) > 0]\n",
    "\n",
    "# prepare the data to fit the vector representation. Returns a generator \n",
    "def prepare_data(infile, w2v):\n",
    "    with open(infile, 'r', encoding='utf-8') as f:\n",
    "        data = []\n",
    "        for line in f:\n",
    "            data.append(np.array(sentence_to_embeddings(preprocess(line), w2v)))\n",
    "        return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v = load_embeddings(os.path.join(DATA_PATH, 'GoogleNews-vectors-negative300.bin'), binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2737, 300)"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "infile = 'parsed-eurydice.txt'\n",
    "_, _, word2embeddings, vocab_shape = build_vocabulary(infile, w2v)\n",
    "vocab_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [],
   "source": [
    "def window(seq, n=3, step=1):\n",
    "    \"Returns a sliding window (of width n) over data from the iterable\"\n",
    "    \"   s -> (s[0],...s[n-1]), (s[0+skip_n],...,s[n-1+skip_n]), ...   \"\n",
    "    it = iter(seq)\n",
    "    result = tuple(islice(it, n))\n",
    "    if len(result) == n:\n",
    "        yield result    \n",
    "    \n",
    "    result = result[step:]\n",
    "    for elem in it:\n",
    "        result = result + (elem,)\n",
    "        if len(result) == n:\n",
    "            yield result\n",
    "            result = result[step:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a batch generator\n",
    "class BatchGenerator(object):\n",
    "    def __init__(self, data, batch_size=1, sample_dim=None):\n",
    "        self.data = np.array(data)\n",
    "        self.batch_size = batch_size\n",
    "        self.PAD = 0.0\n",
    "        self.EOS = np.ones((1,sample_dim))\n",
    "        \n",
    "    def generate_batch(self):\n",
    "        # every three lines comprise a sample sequence where the first two items\n",
    "        # are the input and the last one is the output\n",
    "        i  = 1\n",
    "        x1 = []\n",
    "        x2 = []\n",
    "        y  = []\n",
    "        while True:\n",
    "            for scene,command,reply in window(self.data, n=3, step=2):                \n",
    "                encoder_input  = np.concatenate([scene, command, self.EOS])\n",
    "                decoder_input  = reply\n",
    "                decoder_output = np.concatenate([reply[1:], self.EOS])\n",
    "                \n",
    "                x1.append(encoder_input)\n",
    "                x2.append(decoder_input)\n",
    "                y.append(decoder_output)\n",
    "                if i == self.batch_size:\n",
    "                    if self.batch_size > 1:\n",
    "                        # pad and return the batch\n",
    "                        x1 = sequence.pad_sequences(x1, padding='post', value=self.PAD)\n",
    "                        x2 = sequence.pad_sequences(x2, padding='post', value=self.PAD)\n",
    "                        y  = sequence.pad_sequences(y, padding='post', value=self.PAD) \n",
    "                    \n",
    "                    yield ([np.array(x1), np.array(x2)], np.array(y))\n",
    "                    \n",
    "                    i  = 1\n",
    "                    x1 = []\n",
    "                    x2 = []\n",
    "                    y  = []\n",
    "                else:\n",
    "                    i += 1\n",
    "            # return the empty list if we don't have enough elements for a full sequence input-output sequence\n",
    "            yield []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_generator = BatchGenerator(prepare_data(infile, word2embeddings), batch_size=16, sample_dim=w2v.vector_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns train, inference_encoder and inference_decoder models\n",
    "def define_models(hidden_dim, output_dim, n_units):\n",
    "    \"\"\"\n",
    "    @hidden_dim the dimensionality of the input (e.g. 300 for standard word2vec embeddings)\n",
    "    @n_output the dimensionality of the output (e.g. the vocabulary size)\n",
    "    @n_units the number of LSTM units in the encoded/decoder\n",
    "    \"\"\"\n",
    "    # define training encoder. We use return_state to retrieve the hidden states for the encoder and\n",
    "    # provide them as input to the decoder\n",
    "    encoder_inputs = Input(shape=(None, hidden_dim)) # timesteps, features\n",
    "    masking = Masking(mask_value=0.)(encoder_inputs)\n",
    "    encoder = LSTM(n_units, return_state=True)\n",
    "    encoder_outputs, state_h, state_c = encoder(masking)\n",
    "    encoder_states = [state_h, state_c]\n",
    "    \n",
    "    # define training decoder. It is initialized with the encoder hidden states\n",
    "    decoder_inputs = Input(shape=(None, output_dim))\n",
    "    decoder_lstm = LSTM(n_units, return_sequences=True, return_state=True)\n",
    "    decoder_outputs, _, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states)\n",
    "    decoder_dense = Dense(output_dim, activation='tanh')\n",
    "    decoder_outputs = decoder_dense(decoder_outputs)\n",
    "    model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    "    \n",
    "    # define inference encoder\n",
    "    encoder_model = Model(encoder_inputs, encoder_states)\n",
    "    \n",
    "    # define inference decoder\n",
    "    decoder_state_input_h = Input(shape=(n_units,))\n",
    "    decoder_state_input_c = Input(shape=(n_units,))\n",
    "    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "    decoder_outputs, state_h, state_c = decoder_lstm(decoder_inputs, initial_state=decoder_states_inputs)\n",
    "    decoder_states = [state_h, state_c]\n",
    "    decoder_outputs = decoder_dense(decoder_outputs)\n",
    "    decoder_model = Model([decoder_inputs] + decoder_states_inputs, [decoder_outputs] + decoder_states)\n",
    "    \n",
    "    # return all models\n",
    "    return model, encoder_model, decoder_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_111 (InputLayer)          (None, None, 300)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "masking_32 (Masking)            (None, None, 300)    0           input_111[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_112 (InputLayer)          (None, None, 300)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_67 (LSTM)                  [(None, 64), (None,  93440       masking_32[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_68 (LSTM)                  [(None, None, 64), ( 93440       input_112[0][0]                  \n",
      "                                                                 lstm_67[0][1]                    \n",
      "                                                                 lstm_67[0][2]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, None, 300)    19500       lstm_68[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 206,380\n",
      "Trainable params: 206,380\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model, encinf, decinf = define_models(300, 300, 64)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='rmsprop')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "5/5 [==============================] - 10s 2s/step - loss: 24.9815\n",
      "Epoch 2/10\n",
      "5/5 [==============================] - 3s 533ms/step - loss: 19.6581\n",
      "Epoch 3/10\n",
      "5/5 [==============================] - 3s 558ms/step - loss: 15.7342\n",
      "Epoch 4/10\n",
      "2/5 [===========>..................] - ETA: 1s - loss: 14.2608"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "all the input arrays must have same number of dimensions",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-466-77de570bfac2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_generator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_generator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgenerate_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m# Save model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDATA_PATH\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m's2s.h5'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\conda\\conda\\envs\\ml\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 91\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\conda\\conda\\envs\\ml\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   2190\u001b[0m                 \u001b[0mbatch_index\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2191\u001b[0m                 \u001b[1;32mwhile\u001b[0m \u001b[0msteps_done\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2192\u001b[1;33m                     \u001b[0mgenerator_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput_generator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2193\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2194\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'__len__'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\conda\\conda\\envs\\ml\\lib\\site-packages\\keras\\utils\\data_utils.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    791\u001b[0m             \u001b[0msuccess\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    792\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0msuccess\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 793\u001b[1;33m                 \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\conda\\conda\\envs\\ml\\lib\\site-packages\\six.py\u001b[0m in \u001b[0;36mreraise\u001b[1;34m(tp, value, tb)\u001b[0m\n\u001b[0;32m    691\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    692\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 693\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    694\u001b[0m         \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    695\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\conda\\conda\\envs\\ml\\lib\\site-packages\\keras\\utils\\data_utils.py\u001b[0m in \u001b[0;36m_data_generator_task\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    656\u001b[0m                             \u001b[1;31m# => Serialize calls to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    657\u001b[0m                             \u001b[1;31m# infinite iterator/generator's next() function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 658\u001b[1;33m                             \u001b[0mgenerator_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_generator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    659\u001b[0m                             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgenerator_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    660\u001b[0m                         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-462-1f4dc883ab63>\u001b[0m in \u001b[0;36mgenerate_batch\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     19\u001b[0m                 \u001b[0mencoder_input\u001b[0m  \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mscene\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcommand\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mEOS\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m                 \u001b[0mdecoder_input\u001b[0m  \u001b[1;33m=\u001b[0m \u001b[0mreply\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m                 \u001b[0mdecoder_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mreply\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mEOS\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m                 \u001b[0mx1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mencoder_input\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: all the input arrays must have same number of dimensions"
     ]
    }
   ],
   "source": [
    "model.fit_generator(batch_generator.generate_batch(), steps_per_epoch=5, epochs=10)\n",
    "\n",
    "# Save model\n",
    "model.save(os.path.join(DATA_PATH, 's2s.h5'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml (3.6)",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}