Pytorch_VAE_Conv.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pytorch_VAE_Conv.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arkaprabha10/Samsung-PRISM/blob/main/Pytorch_VAE_Conv.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EUEMlX1jCMth"
      },
      "source": [
        "import os\n",
        "import torch\n",
        "from torch.autograd import Variable\n",
        "from torch.nn import Sequential, Conv2d, Linear, BatchNorm2d, Dropout, Softmax, Tanh, BCELoss, ReLU, Module, ConvTranspose2d, Flatten, Sigmoid\n",
        "from torch.optim import Adam\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import librosa.display\n",
        "import IPython.display as ipd\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jHDoIWwHWjox"
      },
      "source": [
        "input_length = 22050*5\n",
        "def load_audio_file(file_path, input_length=input_length):\n",
        "    data = librosa.core.load(file_path, sr=22050)[0] #, sr=16000\n",
        "    if len(data)>input_length:\n",
        "        \n",
        "        \n",
        "        max_offset = len(data)-input_length\n",
        "        \n",
        "        offset = np.random.randint(max_offset)\n",
        "        \n",
        "        data = data[offset:(input_length+offset)]\n",
        "        \n",
        "        \n",
        "    else:\n",
        "        if input_length > len(data):\n",
        "            max_offset = input_length - len(data)\n",
        "\n",
        "            offset = np.random.randint(max_offset)\n",
        "        else:\n",
        "            offset = 0\n",
        "        \n",
        "        \n",
        "        data = np.pad(data, (offset, input_length - len(data) - offset), \"constant\")\n",
        "        \n",
        "        \n",
        "    data = preprocess_audio_mel_T(data)\n",
        "    return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l7htwoR_Xgk_"
      },
      "source": [
        "n_mels = 80\n",
        "def preprocess_audio_mel_T(signal, sample_rate=22050, window_size=20, #log_specgram\n",
        "                 step_size=10, eps=1e-10):\n",
        "    # pre_emphasis = 0.97\n",
        "    # emphasized_signal = np.append(signal[0], signal[1:] - pre_emphasis * signal[:-1])\n",
        "    # frame_size = 0.025\n",
        "    # frame_stride = 0.01\n",
        "    # frame_length, frame_step = frame_size * sample_rate, frame_stride * sample_rate  # Convert from seconds to samples\n",
        "    # signal_length = len(emphasized_signal)\n",
        "    # frame_length = int(round(frame_length))\n",
        "    # frame_step = int(round(frame_step))\n",
        "    # num_frames = int(np.ceil(float(np.abs(signal_length - frame_length)) / frame_step))  # Make sure that we have at least 1 frame\n",
        "\n",
        "    # pad_signal_length = num_frames * frame_step + frame_length\n",
        "    # z = np.zeros((pad_signal_length - signal_length))\n",
        "    # pad_signal = np.append(emphasized_signal, z)\n",
        "    # for i in range (0,len(pad_signal),frame_step):\n",
        "    #   temp=pad_signal[i:frame_length+i]\n",
        "    #   temp*=np.hanning(len(temp))\n",
        "    #   for j in range(len(temp)):\n",
        "    #     pad_signal[i+j]=temp[j]\n",
        "\n",
        "    # ans=librosa.core.stft(y=signal, win_length=int(0.025*sample_rate), hop_length=int(0.00971*sample_rate), n_fft = 2046)\n",
        "    ans = librosa.feature.melspectrogram(signal, sr = sample_rate, n_mels = 80)\n",
        "    return ans"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "20lckJKBXmKT",
        "outputId": "d2fbf7cc-b043-463a-cfac-57d18d0cd437"
      },
      "source": [
        "x_in = []\n",
        "count_dir = 0\n",
        "count_file = 0\n",
        "Data_Path = '/content/drive/MyDrive/speech_data'\n",
        "with os.scandir(Data_Path) as entries:\n",
        "  # for entry in entries:\n",
        "  #   if count_dir == 1:\n",
        "  #     break\n",
        "  #   count_dir +=1\n",
        "#   #     with os.scandir(entry) as d:\n",
        "#   #       if d.is_dir():\n",
        "    # with os.scandir(entry) as files:\n",
        "  for file_ in entries:\n",
        "    if count_file == 30:\n",
        "      break\n",
        "    count_file+=1\n",
        "    data = load_audio_file(file_)\n",
        "    x_in.append(data.tolist())\n",
        "print('Data loaded successfully!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Data loaded successfully!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3EVt2cYqYAac"
      },
      "source": [
        "x_in = np.expand_dims(x_in, -1).astype('float32')\n",
        "max = np.amax(x_in)\n",
        "mean = np.mean(x_in)\n",
        "x_in = x_in/max"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aL-nBZSYOwb"
      },
      "source": [
        "x_in = x_in - mean/max"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fc8RBSVEYrfN"
      },
      "source": [
        "x_train = x_in[:12]\n",
        "print(x_train.shape)\n",
        "x_test = x_in[12:]\n",
        "print(x_test.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vk1Vizy5ZupU",
        "outputId": "570e1ca1-9161-406d-e85a-f8461e6e7f53"
      },
      "source": [
        "x_train = torch.from_numpy(x_train)\n",
        "x_test = torch.from_numpy(x_test)\n",
        "print(x_train.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([12, 80, 216, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FJmM1xuuaV9t",
        "outputId": "82919936-e1d7-40d9-c767-1f53baa0d7e3"
      },
      "source": [
        "train_x = x_train\n",
        "test_x = x_test\n",
        "\n",
        "print(train_x.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([12, 80, 216, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BIQF6COLbomq"
      },
      "source": [
        "class VAE(Module):\n",
        "  def __init__(self):\n",
        "    super(VAE, self).__init__()\n",
        "\n",
        "    #encoder\n",
        "    self.enc1= Sequential(\n",
        "        Conv2d(80, 128, kernel_size=(3,1), stride = (1,1)),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(128),\n",
        "        Conv2d(128, 256, kernel_size=(3,1), stride = (2,1)),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(256),\n",
        "        Conv2d(256, 512, kernel_size=(3,1), stride = (2,1)),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(512),\n",
        "        Flatten()\n",
        "    )\n",
        "    self.enc2 = Sequential(\n",
        "        Linear(in_features=512, out_features=128),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(128),\n",
        "    )\n",
        "     \n",
        "    def Sampling(self, mu, log_var):\n",
        "      std = torch.exp(0.5*log_var) #std_deviation\n",
        "      eps = torch.randn_like(std)\n",
        "      sample = mu + (eps*std)\n",
        "      return sample\n",
        "    \n",
        "    #decoder\n",
        "    self.dec1 = Sequential(\n",
        "        Linear(in_features=128, out_features=20*54*256),\n",
        "        BatchNorm2d(20*54*256),\n",
        "    )\n",
        "    self.dec2 = Sequential(\n",
        "        Conv2d(256, 128, kernel_size=(3,1), stride = (2,1)),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(128),\n",
        "        Conv2d(128, 64, kernel_size=(3,1), stride = (2,1)),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(64),\n",
        "        Conv2d(64, 1, kernel_size=(1,3), stride = (1,1)),\n",
        "        ReLU(inplace=True),\n",
        "        BatchNorm2d(1),\n",
        "    )\n",
        "    \n",
        "  def forward(self, x):\n",
        "    x = self.enc1(x)\n",
        "    x = self.enc2(x).view(-1, 2, 128)\n",
        "\n",
        "    mu = x[:, 0, :]\n",
        "\n",
        "    log_var = x[:, 1, :]\n",
        "\n",
        "    z = self.Sampling(mu, log_var)\n",
        "\n",
        "    x = self.dec1(z)\n",
        "    x = torch.reshape(x,(20,54,256))\n",
        "    reconstruction = self.dec2(x)\n",
        "\n",
        "    return reconstruction, mu, log_var"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4MKY6iuacP5b",
        "outputId": "10c4922f-6d69-42bd-e428-23026f25b1a6"
      },
      "source": [
        "model = VAE()\n",
        "# defining the optimizer\n",
        "optimizer = Adam(model.parameters(), lr=1e-4)\n",
        "# defining the loss function\n",
        "criterion = BCELoss()\n",
        "# checking if GPU is available\n",
        "if torch.cuda.is_available():\n",
        "    model = model.cuda()\n",
        "    criterion = criterion.cuda()\n",
        "    \n",
        "print(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "VAE(\n",
            "  (enc1): Sequential(\n",
            "    (0): Conv2d(80, 128, kernel_size=(3, 1), stride=(1, 1))\n",
            "    (1): ReLU(inplace=True)\n",
            "    (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (3): Conv2d(128, 256, kernel_size=(3, 1), stride=(2, 1))\n",
            "    (4): ReLU(inplace=True)\n",
            "    (5): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (6): Conv2d(256, 512, kernel_size=(3, 1), stride=(2, 1))\n",
            "    (7): ReLU(inplace=True)\n",
            "    (8): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (9): Flatten(start_dim=1, end_dim=-1)\n",
            "  )\n",
            "  (enc2): Sequential(\n",
            "    (0): Linear(in_features=512, out_features=128, bias=True)\n",
            "    (1): ReLU(inplace=True)\n",
            "    (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  )\n",
            "  (dec1): Sequential(\n",
            "    (0): Linear(in_features=128, out_features=276480, bias=True)\n",
            "    (1): BatchNorm2d(276480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  )\n",
            "  (dec2): Sequential(\n",
            "    (0): Conv2d(256, 128, kernel_size=(3, 1), stride=(2, 1))\n",
            "    (1): ReLU(inplace=True)\n",
            "    (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (3): Conv2d(128, 64, kernel_size=(3, 1), stride=(2, 1))\n",
            "    (4): ReLU(inplace=True)\n",
            "    (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (6): Conv2d(64, 1, kernel_size=(1, 3), stride=(1, 1))\n",
            "    (7): ReLU(inplace=True)\n",
            "    (8): BatchNorm2d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3goqmdzwYxFm"
      },
      "source": [
        "def train(epoch):\n",
        "    model.train()\n",
        "    tr_loss = 0\n",
        "    # getting the training set\n",
        "    x_train, y_train = Variable(train_x), Variable(train_x)\n",
        "    # getting the validation set\n",
        "    x_val, y_val = Variable(test_x), Variable(test_x)\n",
        "    # converting the data into GPU format\n",
        "    if torch.cuda.is_available():\n",
        "        x_train = x_train.cuda()\n",
        "        y_train = y_train.cuda()\n",
        "        x_val = x_val.cuda()\n",
        "        y_val = y_val.cuda()\n",
        "\n",
        "    # clearing the Gradients of the model parameters\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # prediction for training and validation set\n",
        "    output_train = model(x_train)\n",
        "    output_val = model(x_val)\n",
        "\n",
        "    # computing the training and validation loss\n",
        "    loss_train = criterion(output_train, y_train)\n",
        "    loss_val = criterion(output_val, y_val)\n",
        "    train_losses.append(loss_train)\n",
        "    val_losses.append(loss_val)\n",
        "\n",
        "    # computing the updated weights of all the model parameters\n",
        "    loss_train.backward()\n",
        "    optimizer.step()\n",
        "    tr_loss = loss_train.item()\n",
        "    if epoch%2 == 0:\n",
        "        # printing the validation loss\n",
        "        print('Epoch : ',epoch+1, '\\t', 'loss :', loss_val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 355
        },
        "id": "sSaYNVdAZV9K",
        "outputId": "287c049f-0ece-4b9c-bb08-97dca4b0d2f5"
      },
      "source": [
        "n_epochs = 25\n",
        "# empty list to store training losses\n",
        "train_losses = []\n",
        "# empty list to store validation losses\n",
        "val_losses = []\n",
        "# training the model\n",
        "for epoch in range(n_epochs):\n",
        "    train(epoch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-49-2335eb460d1e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# training the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-48-60fa32fbd246>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(epoch)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0;31m# prediction for training and validation set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0moutput_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0moutput_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-46-f8ca1460b09f>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     47\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menc1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menc2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mmu\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1688\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1689\u001b[0m         \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1690\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1691\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1692\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (12x26624 and 512x128)"
          ]
        }
      ]
    }
  ]
}