cond_wand_transformer_final.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LYszVig1i9HZ",
        "outputId": "824f865d-2119-424e-f892-c89f3ee3a9a5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m48.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m25.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.14.1 tokenizers-0.13.3 transformers-4.28.1\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "#!pip install tensorflow-addons\n",
        "#!pip install tf-transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ECsOBv4AIvjk"
      },
      "outputs": [],
      "source": [
        "import tensorflow\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.layers import DepthwiseConv1D, SeparableConv1D\n",
        "#from tensorflow.keras.utils import Transformer, MultiHeadAttention, LayerNormalization, Dropout\n",
        "import transformers\n",
        "from transformers import BertConfig\n",
        "from google.colab import drive\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from transformers.models.bert.modeling_tf_bert import TFBertEncoder, TFBertEmbeddings\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "#from tensorflow_addons.layers import TransformerEncoder\n",
        "#import tf_transformers as tft\n",
        "#from tf_transformers.layers import TransformerEncoder\n",
        "#import tensorflow_addons as tfa"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3jlZuVs86I2b",
        "outputId": "de76dd39-79ad-437f-c636-f9238adbf01b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "drive.mount('/content/drive')\n",
        "data_path = '/content/drive/MyDrive/Columbia Spring 2023/Embedded AI/Conductor_wand/'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "gJxD-s4Wmiro"
      },
      "outputs": [],
      "source": [
        "# HELPER CLASSES\n",
        "## VECTOR MAGNITUDE LAYER FOR IMU FUSION\n",
        "class vectorMagFusion(tf.keras.layers.Layer):\n",
        "  def __init__(self):\n",
        "    super(vectorMagFusion, self).__init__()\n",
        "  def build(self, input_shape):\n",
        "    super(vectorMagFusion, self).build(input_shape)\n",
        "  def call(self, inputs):\n",
        "    return tf.norm(inputs,axis=-1)\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return (input_shape[0],1)\n",
        "\n",
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "  matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
        "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "  scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "  if mask is not None:\n",
        "    scaled_attention_logits += (mask * -1e9)\n",
        "  attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
        "  output = tf.matmul(attention_weights, v)\n",
        "  return output, attention_weights\n",
        "\n",
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "    # CHECK IF MODEL SIZING IS APPROPRIATE\n",
        "    assert d_model % num_heads == 0\n",
        "    # CALCULATE MODEL DEPTH\n",
        "    self.depth = d_model//self.num_heads\n",
        "    # QUERIES, KEYS, AND VALUES - LINEAR LAYERS TO MODEL LEARNABLE WEIGHTS\n",
        "    self.wq = tf.keras.layers.Dense(d_model)\n",
        "    self.wk = tf.keras.layers.Dense(d_model)\n",
        "    self.wv = tf.keras.layers.Dense(d_model)\n",
        "    # BUILD DENSE PATH\n",
        "    self.dense = tf.keras.layers.Dense(d_model)\n",
        "  def split_heads(self, x, batch_size):\n",
        "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth)) # split last dimension into num heads and depth\n",
        "    return tf.transpose(x,perm=[0,2,1,3]) # (batch_size, num_heads, seq_len, depth)\n",
        "\n",
        "  def call(self, v, k, q, mask):\n",
        "    batch_size = tf.shape(q)[0]\n",
        "    # SPLIT AND RESHAPE Qs, Ks, and Vs INTO MULTIPLE HEADS\n",
        "    q = self.wq(q)\n",
        "    k = self.wk(k)\n",
        "    v = self.wv(v)\n",
        "    q = self.split_heads(q, batch_size) # (batchSize, numHeads, seqLen, depth)\n",
        "    k = self.split_heads(k, batch_size) # (batchSize, numHeads, seqLen, depth)\n",
        "    v = self.split_heads(v, batch_size) # (batchSize, numHeads, seqLen, depth)\n",
        "    # PERFORM SCALED, ATTENTIONAL MULTIPLICATION OF Qs Ks and Vs\n",
        "    scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask) \n",
        "    # TRANSPOSE AND 'CONCATENATE/SQUEEZE' \n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0,2,1,3]) # (batchSize, seqLen, numHeads, depth)\n",
        "    concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model)) # (batchSize, seqLen, depth)\n",
        "    # FEEDFORWARD OUTPUT \n",
        "    output = self.dense(concat_attention) # (batch_size, seq_len, d_model)\n",
        "    return output, attention_weights\n",
        "  \n",
        "  def get_config(self):\n",
        "    config = super().get_config().copy()\n",
        "    config.update({\n",
        "        'depth' : self.depth,\n",
        "        'wq' : self.wq,\n",
        "        'wk' : self.wk,\n",
        "        'wv' : self.wv,\n",
        "        'dense' : self.dense,\n",
        "    })\n",
        "    return config\n",
        "    \n",
        "def point_wise_feed_forward_network(out_size, dff):\n",
        "  return tf.keras.Sequential([\n",
        "      tf.keras.layers.Dense(dff, activation='relu'),\n",
        "      # tf.keras.layers.Dense(d_model)\n",
        "      tf.keras.layers.Dense(out_size)\n",
        "  ])\n",
        "  \n",
        "class PositionalEncoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, position_len, d_model):\n",
        "    super().__init__()\n",
        "    self.pos_encoding = self.positional_encoding(position_len, d_model)     \n",
        "\n",
        "  def positional_encoding(self, position_len, d_model):\n",
        "    positions = np.arange(position_len)[:,np.newaxis]\n",
        "    d_idxs = np.arange(d_model)[np.newaxis,:]\n",
        "    theta_arr = positions * (1 / (np.power(10000,(2*(d_idxs//2))/np.float32(d_model))))\n",
        "    sines = np.sin(theta_arr[:, 0::2])\n",
        "    cosines = np.cos(theta_arr[:, 1::2])\n",
        "\n",
        "    # concatenate sines and cosines into an embedding sized matrix\n",
        "    pos_encoding = np.concatenate([sines, cosines],axis=-1)\n",
        "    pos_encoding = pos_encoding[np.newaxis,...]\n",
        "    return tf.cast(pos_encoding, dtype=tf.float32)\n",
        "\n",
        "  def call(self, x):\n",
        "    # print(tf.shape(self.pos_encoding[:, :tf.shape(inputs)[1],:]))\n",
        "    return x + self.pos_encoding[:, :tf.shape(x)[1],:]\n",
        "\n",
        "class TransformerEncoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, out_size, rate=0.1):\n",
        "      super(TransformerEncoder, self).__init__()        \n",
        "      self.d_model = d_model\n",
        "      self.num_heads = num_heads\n",
        "      self.dff = dff\n",
        "      self.rate = rate\n",
        "      self.out_size = out_size\n",
        "      \n",
        "      self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "      self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "      \n",
        "\n",
        "      self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "      self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "      \n",
        "      self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "      self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "      \n",
        "  def call(self, x, training=None):\n",
        "      attn_output, _ = self.mha(x, x, x, None)\n",
        "      attn_output = self.dropout1(attn_output, training=training)\n",
        "      out1 = self.layernorm1(x + attn_output)\n",
        "      ffn_output = self.ffn(out1)\n",
        "      # print('ffn_out1: ',np.shape(ffn_output))\n",
        "      ffn_output = self.dropout2(ffn_output, training=training)\n",
        "      out2 = self.layernorm2(out1 + ffn_output)\n",
        "      # print('transf out: ', np.shape(out2))\n",
        "      return out2\n",
        "\n",
        "class TransformerEncoderOut(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, out_size, rate=0.1):\n",
        "      super(TransformerEncoderOut, self).__init__()        \n",
        "      self.d_model = d_model\n",
        "      self.num_heads = num_heads\n",
        "      self.dff = dff\n",
        "      self.rate = rate\n",
        "      self.out_size = out_size\n",
        "      \n",
        "      self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "      self.ffn = point_wise_feed_forward_network(out_size, dff)\n",
        "      \n",
        "\n",
        "      self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "      self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "      \n",
        "      self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "      self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "      \n",
        "  def call(self, x, training=None):\n",
        "      attn_output, _ = self.mha(x, x, x, None)\n",
        "      attn_output = self.dropout1(attn_output, training=training)\n",
        "      out1 = self.layernorm1(x + attn_output)\n",
        "      ffn_output = self.ffn(out1)\n",
        "      # print('ffn_out1: ',np.shape(ffn_output))\n",
        "      ffn_output = self.dropout2(ffn_output, training=training)\n",
        "      out2 = self.layernorm2(ffn_output)\n",
        "      # print('transf out: ', np.shape(out2))\n",
        "      return out2\n",
        "\n",
        "  def get_config(self):\n",
        "    config = super().get_config().copy()\n",
        "    config.update({\n",
        "        'num_heads': self.num_heads,\n",
        "        'rate' : self.rate,\n",
        "        'd_model' : self.d_model,\n",
        "        'num_heads' : self.num_heads,\n",
        "        'dropout1' : self.dropout1,\n",
        "        'dropout2' : self.dropout2,\n",
        "        'layernorm1' : self.layernorm1,\n",
        "        'layernorm2' : self.layernorm2,\n",
        "        'mha' : self.mha,\n",
        "        'ffn' : self.ffn\n",
        "    })\n",
        "    return config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q4FSDEEPxkBd",
        "outputId": "c6f2c126-8846-4bed-be12-208faf06c137"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "KerasTensor(type_spec=TensorSpec(shape=(3,), dtype=tf.int32, name=None), inferred_value=[None, 238, 6], name='tf.compat.v1.shape_40/Shape:0', description=\"created by layer 'tf.compat.v1.shape_40'\")\n",
            "flat1:  KerasTensor(type_spec=TensorSpec(shape=(2,), dtype=tf.int32, name=None), inferred_value=[None, 2856], name='tf.compat.v1.shape_41/Shape:0', description=\"created by layer 'tf.compat.v1.shape_41'\")\n",
            "layerNorm:  KerasTensor(type_spec=TensorSpec(shape=(3,), dtype=tf.int32, name=None), inferred_value=[None, 238, 12], name='tf.compat.v1.shape_42/Shape:0', description=\"created by layer 'tf.compat.v1.shape_42'\")\n"
          ]
        }
      ],
      "source": [
        "# TRANSFORMER MODEL\n",
        "## HYPER PARAMETERS\n",
        "num_layers = 3 # three encoder layers stacked\n",
        "d_model = 6 # dimensionality of transformer embedding matches number of IMU axes (potential performance bottleneck)\n",
        "dff = 15 # number of hidden FF layers in a transformer module\n",
        "num_heads = 3 # three attention heads per layer (dim / num_heads = 2)\n",
        "dropout_rate = 0.5\n",
        "rate = dropout_rate\n",
        "Fs = 119\n",
        "num_seconds = 2\n",
        "seq_len = Fs * num_seconds\n",
        "num_axes = 6\n",
        "vocab_size= len(range(60,125,5))\n",
        "input_shape = (238,32)\n",
        "num_filters = 6\n",
        "out_size = 13\n",
        "## ARCHITECTURE\n",
        "config = BertConfig(vocab_size=vocab_size,hidden_size=d_model,num_hidden_layers=num_layers, num_attention_heads=num_heads, max_position_embeddings=seq_len, output_attentions=False, output_hidden_states=False)\n",
        "inputs = tf.keras.layers.Input(shape=(seq_len,num_axes), dtype=tf.float16)\n",
        "# vmLayer = vectorMagFusion()(inputs)\n",
        "# vmLayer = tf.cast(vmLayer, tf.int32)\n",
        "print(tf.shape(inputs))\n",
        "\n",
        "# CONVOLUTION LAYER\n",
        "convLayer1 = SeparableConv1D(\n",
        "    filters = d_model*2,\n",
        "    kernel_size = 3,\n",
        "    strides=1,\n",
        "    padding='same',\n",
        "    activation='relu',\n",
        "    input_shape=(seq_len,num_axes)\n",
        ")(inputs)\n",
        "\n",
        "# TRANSFORMER ENCODER \n",
        "posEncodeLayer = PositionalEncoder(seq_len, d_model*2)(convLayer1)\n",
        "encodeLayer = TransformerEncoder(d_model*2, num_heads, dff, out_size, rate)(posEncodeLayer)\n",
        "for i in range(1,num_layers):  \n",
        "  encodeLayer = TransformerEncoder(d_model*2, num_heads, dff, out_size, rate)(encodeLayer)\n",
        "\n",
        "# LayerNorm \n",
        "layerNorm = tf.keras.layers.LayerNormalization(axis=-1, center=True, scale=True)(encodeLayer)\n",
        "flat1 = tf.keras.layers.Flatten(input_shape=(seq_len,num_axes))(layerNorm)\n",
        "# FLATTEN\n",
        "print('flat1: ',tf.shape(flat1))\n",
        "print('layerNorm: ',tf.shape(layerNorm))\n",
        "\n",
        "dense1 = tf.keras.layers.Dense(714, activation='gelu')(flat1)\n",
        "dense2 = tf.keras.layers.Dense(357, activation='gelu')(dense1)\n",
        "dense3 = tf.keras.layers.Dense(13, activation='softmax')(dense2)\n",
        "\n",
        "# outFinal = tf.squeeze(out2,axis=2)\n",
        "# print(tf.shape(outFinal))\n",
        "TFmodel = tf.keras.Model(inputs=inputs, outputs=dense3)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "eoFMdO0VIySi"
      },
      "outputs": [],
      "source": [
        "data = {}\n",
        "NUM_CLASSES = 13\n",
        "x_train, y_train, x_test, y_test = [], [], [], []\n",
        "split = 47\n",
        "Fs = 119\n",
        "data_len = 60 # 60 seconds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1rBvvR8I06P",
        "outputId": "88db814b-7655-4b1d-eb0f-2f1e45866a07"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(611, 238, 6)\n",
            "(611,)\n",
            "(156, 238, 6)\n",
            "(156,)\n"
          ]
        }
      ],
      "source": [
        "#uncomment if you've uploaded a ZIP file, and change the folder\n",
        "#!unzip condSamples.zip\n",
        "scaler = StandardScaler()\n",
        "for i in range(60,125,5):\n",
        "    data[i] = np.loadtxt(data_path + str(i)+\"bpm3.csv\",skiprows=1,delimiter=',')[0:119*60,1:]\n",
        "\n",
        "for i, array in data.items():\n",
        "    # print(np.shape(array))\n",
        "    scaler.fit(array)\n",
        "    normArray = scaler.transform(array)\n",
        "    for j in range(0, 59):\n",
        "        if j < split:\n",
        "            x_train.append(normArray[j*119:(j+2)*119])\n",
        "            y_train.append(i)\n",
        "            # y_train.append(np.ones((Fs*2)) * i)\n",
        "            #print(i)\n",
        "        else:\n",
        "            x_test.append(array[j*119:(j+2)*119])\n",
        "            # y_test.append(np.ones((Fs*2)) * i)\n",
        "            y_test.append(i)\n",
        "#print(x_test)\n",
        "x_train = np.array(x_train)\n",
        "y_train = np.array(y_train)\n",
        "x_test = np.array(x_test)\n",
        "y_test = np.array(y_test)\n",
        "print(x_train.shape)\n",
        "print(y_train.shape)\n",
        "print(x_test.shape)\n",
        "print(y_test.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s1C-oPzWMpTs",
        "outputId": "9a255d74-049b-470b-d8dd-1817ab7c0afb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from tensorflow.keras.optimizers import Adam\n",
        "enc = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
        "# enc = enc.fit(y_train.reshape(-1,1))\n",
        "enc = enc.fit(y_train.reshape(-1,1))\n",
        "y_train_enc = enc.transform(y_train.reshape(-1,1))\n",
        "# y_train = enc.transform(y_train.reshape(-1, 1))\n",
        "# y_train_enc = enc.transform(y_train.reshape(-1,1)).reshape(y_train.shape[0],y_train.shape[1],-1)\n",
        "enc = enc.fit(y_test.reshape(-1,1))\n",
        "y_test_enc = enc.transform(y_test.reshape(-1,1))\n",
        "# y_test = enc.transform(y_test.reshape(1, -1))\n",
        "# y_test_enc = enc.transform(y_test.reshape(-1,1)).reshape(y_test.shape[0],y_test.shape[1],-1)\n",
        "lr = 0.001\n",
        "optimizer = Adam(learning_rate=lr)\n",
        "TFmodel.compile(loss='categorical_crossentropy',\n",
        "              optimizer=optimizer, metrics=['acc'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VM-lPIy7PypD",
        "outputId": "6da4d0f4-e8dd-48a6-a6b5-a4005fa8bf2f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(611, 238, 6)\n",
            "(611, 13)\n",
            "(156, 238, 6)\n",
            "(156, 13)\n"
          ]
        }
      ],
      "source": [
        "print(x_train.shape)\n",
        "print(y_train_enc.shape)\n",
        "print(x_test.shape)\n",
        "print(y_test_enc.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hw9WmLInk-BF",
        "outputId": "9b709523-d718-48a5-a838-e2d06c574698"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 1.78133623  1.08477004  0.10670362 -2.19204676 -0.04775503  1.30778738]\n",
            " [ 1.62530535  1.16372788 -0.12255043 -2.35370403 -0.03546806  1.41973908]\n",
            " [ 1.43583929  1.18984926 -0.32729305 -2.56414732  0.01895181  1.50816376]\n",
            " ...\n",
            " [ 1.52269392 -1.14979597  0.52628179 -0.51233384  0.50346156  1.7977756 ]\n",
            " [ 1.36397285 -1.14801497  0.39471304 -0.62807781  0.49644085  1.97219265]\n",
            " [ 1.29133779 -0.96279058 -0.1477828  -0.75721211  0.34722553  2.12146114]]\n"
          ]
        }
      ],
      "source": [
        "print(x_train[0][:])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SuaEZjQfJGV2",
        "outputId": "ebd99550-4bf9-4e7e-a772-915546ad7225"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "9/9 [==============================] - 17s 843ms/step - loss: 2.8805 - acc: 0.0710 - val_loss: 5.3433 - val_acc: 0.0000e+00\n",
            "Epoch 2/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 2.6606 - acc: 0.0838 - val_loss: 5.7925 - val_acc: 0.0000e+00\n",
            "Epoch 3/40\n",
            "9/9 [==============================] - 6s 715ms/step - loss: 2.5726 - acc: 0.0856 - val_loss: 6.0760 - val_acc: 0.0000e+00\n",
            "Epoch 4/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 2.5460 - acc: 0.0838 - val_loss: 6.3153 - val_acc: 0.0000e+00\n",
            "Epoch 5/40\n",
            "9/9 [==============================] - 6s 711ms/step - loss: 2.5312 - acc: 0.0838 - val_loss: 6.5802 - val_acc: 0.0000e+00\n",
            "Epoch 6/40\n",
            "9/9 [==============================] - 10s 1s/step - loss: 2.5070 - acc: 0.1184 - val_loss: 6.3404 - val_acc: 0.0000e+00\n",
            "Epoch 7/40\n",
            "9/9 [==============================] - 7s 728ms/step - loss: 2.4781 - acc: 0.1002 - val_loss: 6.8918 - val_acc: 0.0000e+00\n",
            "Epoch 8/40\n",
            "9/9 [==============================] - 11s 1s/step - loss: 2.4646 - acc: 0.1457 - val_loss: 7.6668 - val_acc: 0.0000e+00\n",
            "Epoch 9/40\n",
            "9/9 [==============================] - 6s 720ms/step - loss: 2.3844 - acc: 0.1566 - val_loss: 8.1185 - val_acc: 0.0000e+00\n",
            "Epoch 10/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 2.3952 - acc: 0.1530 - val_loss: 7.7764 - val_acc: 0.0000e+00\n",
            "Epoch 11/40\n",
            "9/9 [==============================] - 7s 777ms/step - loss: 2.2834 - acc: 0.1658 - val_loss: 9.3757 - val_acc: 0.0000e+00\n",
            "Epoch 12/40\n",
            "9/9 [==============================] - 10s 1s/step - loss: 2.1347 - acc: 0.2077 - val_loss: 11.6196 - val_acc: 0.0000e+00\n",
            "Epoch 13/40\n",
            "9/9 [==============================] - 7s 736ms/step - loss: 2.1187 - acc: 0.2186 - val_loss: 12.6709 - val_acc: 0.0000e+00\n",
            "Epoch 14/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 1.9542 - acc: 0.2368 - val_loss: 14.4828 - val_acc: 0.0000e+00\n",
            "Epoch 15/40\n",
            "9/9 [==============================] - 7s 750ms/step - loss: 1.8474 - acc: 0.2933 - val_loss: 15.3734 - val_acc: 0.0000e+00\n",
            "Epoch 16/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 1.8599 - acc: 0.2714 - val_loss: 16.0754 - val_acc: 0.0000e+00\n",
            "Epoch 17/40\n",
            "9/9 [==============================] - 7s 735ms/step - loss: 1.7427 - acc: 0.3206 - val_loss: 16.8889 - val_acc: 0.0000e+00\n",
            "Epoch 18/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 1.7125 - acc: 0.2878 - val_loss: 15.7989 - val_acc: 0.0161\n",
            "Epoch 19/40\n",
            "9/9 [==============================] - 8s 849ms/step - loss: 1.5734 - acc: 0.3825 - val_loss: 16.2299 - val_acc: 0.0161\n",
            "Epoch 20/40\n",
            "9/9 [==============================] - 9s 934ms/step - loss: 1.5344 - acc: 0.3515 - val_loss: 18.8539 - val_acc: 0.0000e+00\n",
            "Epoch 21/40\n",
            "9/9 [==============================] - 8s 927ms/step - loss: 1.4869 - acc: 0.4189 - val_loss: 17.9059 - val_acc: 0.0323\n",
            "Epoch 22/40\n",
            "9/9 [==============================] - 8s 823ms/step - loss: 1.4110 - acc: 0.4262 - val_loss: 17.3949 - val_acc: 0.0000e+00\n",
            "Epoch 23/40\n",
            "9/9 [==============================] - 8s 870ms/step - loss: 1.3565 - acc: 0.4663 - val_loss: 20.2056 - val_acc: 0.0000e+00\n",
            "Epoch 24/40\n",
            "9/9 [==============================] - 8s 816ms/step - loss: 1.2296 - acc: 0.5100 - val_loss: 18.5075 - val_acc: 0.0806\n",
            "Epoch 25/40\n",
            "9/9 [==============================] - 8s 874ms/step - loss: 1.1351 - acc: 0.5446 - val_loss: 20.5151 - val_acc: 0.0000e+00\n",
            "Epoch 26/40\n",
            "9/9 [==============================] - 8s 809ms/step - loss: 1.0471 - acc: 0.5956 - val_loss: 20.9619 - val_acc: 0.0161\n",
            "Epoch 27/40\n",
            "9/9 [==============================] - 8s 901ms/step - loss: 1.0175 - acc: 0.5847 - val_loss: 20.5004 - val_acc: 0.0323\n",
            "Epoch 28/40\n",
            "9/9 [==============================] - 8s 792ms/step - loss: 0.8945 - acc: 0.6321 - val_loss: 22.9722 - val_acc: 0.0000e+00\n",
            "Epoch 29/40\n",
            "9/9 [==============================] - 8s 895ms/step - loss: 0.8239 - acc: 0.6867 - val_loss: 20.5367 - val_acc: 0.0323\n",
            "Epoch 30/40\n",
            "9/9 [==============================] - 8s 793ms/step - loss: 0.7349 - acc: 0.7213 - val_loss: 20.3702 - val_acc: 0.0806\n",
            "Epoch 31/40\n",
            "9/9 [==============================] - 8s 923ms/step - loss: 0.7153 - acc: 0.7231 - val_loss: 19.1791 - val_acc: 0.0484\n",
            "Epoch 32/40\n",
            "9/9 [==============================] - 7s 764ms/step - loss: 0.6052 - acc: 0.7505 - val_loss: 20.9718 - val_acc: 0.0161\n",
            "Epoch 33/40\n",
            "9/9 [==============================] - 10s 1s/step - loss: 0.5780 - acc: 0.7723 - val_loss: 21.1623 - val_acc: 0.0484\n",
            "Epoch 34/40\n",
            "9/9 [==============================] - 7s 727ms/step - loss: 0.5237 - acc: 0.7851 - val_loss: 21.5151 - val_acc: 0.0484\n",
            "Epoch 35/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 0.5794 - acc: 0.7596 - val_loss: 20.2614 - val_acc: 0.1129\n",
            "Epoch 36/40\n",
            "9/9 [==============================] - 7s 736ms/step - loss: 0.4994 - acc: 0.7942 - val_loss: 18.8422 - val_acc: 0.2097\n",
            "Epoch 37/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 0.4703 - acc: 0.8379 - val_loss: 19.3820 - val_acc: 0.0000e+00\n",
            "Epoch 38/40\n",
            "9/9 [==============================] - 7s 738ms/step - loss: 0.4013 - acc: 0.8670 - val_loss: 20.2484 - val_acc: 0.0323\n",
            "Epoch 39/40\n",
            "9/9 [==============================] - 9s 1s/step - loss: 0.3225 - acc: 0.8798 - val_loss: 22.2456 - val_acc: 0.0645\n",
            "Epoch 40/40\n",
            "9/9 [==============================] - 7s 729ms/step - loss: 0.2986 - acc: 0.8816 - val_loss: 21.3349 - val_acc: 0.0484\n"
          ]
        }
      ],
      "source": [
        "history = TFmodel.fit(\n",
        "    x_train, y_train_enc,\n",
        "    epochs=40,\n",
        "    batch_size=64,\n",
        "    validation_split=0.1,\n",
        "    shuffle=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yulgN1np7QTk"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xmu9SGAIJAvy",
        "outputId": "7f99f0a3-da52-4f5c-c316-cd75a0e4d3a8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "enc = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
        "\n",
        "enc = enc.fit(y_train.reshape(-1, 1))\n",
        "\n",
        "y_train = enc.transform(y_train.reshape(-1, 1))\n",
        "y_test = enc.transform(y_test.reshape(-1, 1))\n",
        "model = keras.Sequential()\n",
        "model.add(\n",
        "    keras.layers.Bidirectional(\n",
        "        keras.layers.LSTM(\n",
        "            units=128,\n",
        "            input_shape=[x_train.shape[1], x_train.shape[2]]\n",
        "        )\n",
        "    )\n",
        ")\n",
        "model.add(keras.layers.Dropout(rate=0.5))\n",
        "model.add(keras.layers.Dense(units=128, activation='relu'))\n",
        "model.add(keras.layers.Dense(y_train.shape[1], activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer='adam', metrics=['acc'])"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "c7ymYL8L5IrK"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}