CLSWGAN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "CLSWGAN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/purbayankar/TransformerCLSWGAN/blob/main/CLSWGAN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ACEQkKrwhtiL",
        "outputId": "c736b67f-f1d2-45fb-81c4-051465a1c173"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XoamzWWKjwGp"
      },
      "source": [
        "import torch\n",
        "from torch.autograd import Variable\n",
        "from torch import autograd\n",
        "import time\n",
        "\n",
        "\n",
        "def calc_gradient_penalty(opt, netD, res_real, res_fake, att):\n",
        "    alpha = torch.rand(opt.length, 1)\n",
        "    alpha = alpha.expand(res_real.size())\n",
        "    if opt.cuda:\n",
        "        alpha = alpha.cuda()\n",
        "    interpolates = alpha * res_real + ((1 - alpha) * res_fake)\n",
        "    interpolates = Variable(interpolates, requires_grad=True)\n",
        "    disc_interpolates = netD(interpolates, Variable(att))\n",
        "    ones = torch.ones(disc_interpolates.size())\n",
        "    if opt.cuda:\n",
        "        ones = ones.cuda()\n",
        "    gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,\n",
        "                              grad_outputs=ones,\n",
        "                              create_graph=True, retain_graph=True, only_inputs=True)[0]\n",
        "    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * opt.lambda1\n",
        "    return gradient_penalty\n",
        "\n",
        "\n",
        "def first_stage_train(opt, models, loaders):\n",
        "    one = torch.tensor(1.)\n",
        "    mone = torch.tensor(-1.)\n",
        "    if opt.cuda:\n",
        "        one = one.cuda()\n",
        "        mone = mone.cuda()\n",
        "    for i in range(opt.first_epoch):\n",
        "        epoch_start_time = time.time()\n",
        "        print('The {}th epoch starts.'.format(i + 1))\n",
        "        for res_real, res_real_class, res_real_att in loaders.img_loader_train:\n",
        "            opt.length = res_real_class.shape[0]\n",
        "            # Train the discriminator.\n",
        "            for param in models.netD.parameters():\n",
        "                param.requires_grad = True\n",
        "            models.netD.zero_grad()\n",
        "            noise = torch.FloatTensor(opt.length, opt.nz_size)\n",
        "            noise.normal_()\n",
        "            if opt.cuda:\n",
        "                res_real_class = res_real_class.cuda()\n",
        "                res_real = res_real.cuda()\n",
        "                noise = noise.cuda()\n",
        "                res_real_att = res_real_att.cuda()\n",
        "            att_dis = Variable(res_real_att, requires_grad=True)\n",
        "            noise_dis = Variable(noise)\n",
        "            res_real_dis = Variable(res_real, requires_grad=True)\n",
        "\n",
        "            dis_real = models.netD(res_real_dis, att_dis)\n",
        "            dis_real_mean = dis_real.mean()\n",
        "            dis_real_mean.backward(mone)\n",
        "\n",
        "            res_fake = models.netG(noise_dis, att_dis)\n",
        "            dis_fake = models.netD(res_fake, att_dis)\n",
        "            dis_fake_mean = dis_fake.mean()\n",
        "            dis_fake_mean.backward(one)\n",
        "\n",
        "            gradient_penalty = calc_gradient_penalty(opt=opt, netD=models.netD, res_real=res_real, res_fake=res_fake,\n",
        "                                                     att=att_dis)\n",
        "            gradient_penalty.backward()\n",
        "            models.optimizerD.step()\n",
        "\n",
        "            if i % 5 == 0:\n",
        "                # Train the generator.\n",
        "                for param in models.netD.parameters():\n",
        "                    param.requires_grad = False\n",
        "\n",
        "                models.netG.zero_grad()\n",
        "\n",
        "                noise.normal_()\n",
        "                noise_gen = Variable(noise)\n",
        "                att_gen = Variable(res_real_att, requires_grad=True)\n",
        "                res_fake = models.netG(noise_gen, att_gen)\n",
        "                dis_fake = models.netD(res_fake, att_gen)\n",
        "                dis_fake_mean = - dis_fake.mean()\n",
        "                cls_result = models.cls(res_fake)\n",
        "                cls_loss = models.cls_criterion(cls_result, res_real_class.squeeze_())\n",
        "\n",
        "                gen_loss = dis_fake_mean + opt.cls_weight * cls_loss\n",
        "                gen_loss.backward()\n",
        "                models.optimizerG.step()\n",
        "\n",
        "        print('This epoch use {} mins {} secs'.format(int((time.time() - epoch_start_time) / 60),\n",
        "                                                      int((time.time() - epoch_start_time) % 60)))\n",
        "\n",
        "\n",
        "def second_stage_train(opt, models, loaders):\n",
        "    for param in models.netG.parameters():\n",
        "        param.requires_grad = False\n",
        "\n",
        "    for i in range(opt.second_epoch):\n",
        "        epoch_start_time = time.time()\n",
        "        print('The {}th epoch starts.'.format(i + 1))\n",
        "        correct_num = 0\n",
        "        complete_num = 0\n",
        "        for _, res_real_class, res_real_att in loaders.img_loader_test:\n",
        "            opt.len_index = res_real_class.shape[0]\n",
        "            models.cls.zero_grad()\n",
        "            noise = torch.FloatTensor(opt.len_index, opt.nz_size)\n",
        "            noise.normal_()\n",
        "            if opt.cuda:\n",
        "                res_real_class = res_real_class.cuda()\n",
        "                noise = noise.cuda()\n",
        "                res_real_att = res_real_att.cuda()\n",
        "            res_real_att = Variable(res_real_att, requires_grad=True)\n",
        "            res_fake = models.netG(noise, res_real_att)\n",
        "            cls_result = models.cls(res_fake)\n",
        "            cls_loss = models.cls_criterion(cls_result, res_real_class.squeeze_())\n",
        "            cls_loss.backward()\n",
        "            models.optimizerC.step()\n",
        "\n",
        "            pred = cls_result.data.max(1)[1]\n",
        "            correct_num += (pred == res_real_class).sum()\n",
        "            complete_num += res_real_class.shape[0]\n",
        "        acc = float(correct_num) / float(complete_num)\n",
        "        print('Post-Training Acc = {}'.format(acc))\n",
        "        print('-----------------------------------------------------')\n",
        "        print('This epoch use {} mins {} secs'.format(int((time.time() - epoch_start_time) / 60),\n",
        "                                                      int((time.time() - epoch_start_time) % 60)))\n",
        "    print('Final Post-Training Acc = {}'.format(acc))\n",
        "\n",
        "\n",
        "def final_test(opt, models, loaders):\n",
        "    correct_num = 0\n",
        "    complete_num = 0\n",
        "    for res_real, res_real_class, res_real_att in loaders.img_loader_test:\n",
        "        opt.len_index = res_real_class.shape[1]\n",
        "        if opt.cuda:\n",
        "            res_real = res_real.cuda()\n",
        "            res_real_class = res_real_class.cuda()\n",
        "        cls_result = models.cls(res_real)\n",
        "        pred = cls_result.data.max(1)[1]\n",
        "        correct_num += (pred == res_real_class.squeeze_()).sum()\n",
        "        complete_num += res_real_class.shape[0]\n",
        "    acc = float(correct_num) / float(complete_num)\n",
        "    print('Final Acc = {}'.format(acc))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V1D4UdpNkAZn"
      },
      "source": [
        "import torch\n",
        "\n",
        "\n",
        "class init_args:\n",
        "    def __init__(self):\n",
        "        self.lr = 0.0001\n",
        "        self.lr_c = 0.001\n",
        "        self.beta1 = 0.5\n",
        "        self.res_size = 2048\n",
        "        self.nz_size = 312\n",
        "        self.nz_res_ratio = 1.\n",
        "        self.att_num = 312\n",
        "        self.class_num = 200\n",
        "        self.pre_train_epoch = 100\n",
        "        self.first_epoch = 1000\n",
        "        self.second_epoch = 400\n",
        "        self.cuda = torch.cuda.is_available()\n",
        "        self.res_path = '/content/drive/MyDrive/xlsa17/data/AWA1/res101.mat'\n",
        "        self.att_path = '/content/drive/MyDrive/xlsa17/data/AWA1/att_splits.mat'\n",
        "        self.shuffle = True\n",
        "        self.batch_size = 2000\n",
        "        self.lambda1 = 10\n",
        "        self.cls_weight = 1\n",
        "        self.length = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qenrpcCRkBUM"
      },
      "source": [
        "from torch import nn\n",
        "from torch.autograd import Variable\n",
        "\n",
        "\n",
        "class classifier(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(classifier, self).__init__()\n",
        "        self.fc = nn.Linear(input_dim, output_dim)\n",
        "        self.lsm = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        output = self.fc(inputs)\n",
        "        output = self.lsm(output)\n",
        "        return output\n",
        "\n",
        "                \n",
        "\n",
        "def pre_train(opt, models, loaders):\n",
        "    for i in range(opt.pre_train_epoch):\n",
        "        correct_num = 0\n",
        "        complete_num = 0\n",
        "        image_class: object\n",
        "        for res_real, res_real_class, _ in loaders.img_loader_train:\n",
        "            models.cls.zero_grad()\n",
        "            if opt.cuda:\n",
        "                res_real = res_real.cuda()\n",
        "                res_real_class = res_real_class.cuda()\n",
        "            res_real = Variable(res_real, requires_grad=True)\n",
        "            cls_result = models.cls(res_real)\n",
        "            cls_loss = models.cls_criterion(cls_result, res_real_class.squeeze_())\n",
        "            cls_loss.backward()\n",
        "            models.optimizerC.step()\n",
        "            pred = cls_result.data.max(1)[1]\n",
        "            correct_num += (pred == res_real_class).sum()\n",
        "            complete_num += res_real_class.shape[0]\n",
        "        acc = float(correct_num) / float(complete_num)\n",
        "        print('Pre-Training Acc = {}'.format(acc))\n",
        "        print('-----------------------------------------------------')\n",
        "    print('Final Pre-Training Acc = {}'.format(acc))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v9OiSKK8k7VQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a0c24f9e-505f-4a42-d036-69084ee38292"
      },
      "source": [
        "!pip install axial_attention\n",
        "import torch.nn as nn\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from axial_attention import AxialAttention\n",
        "\n",
        "\n",
        "def weights_init(m):\n",
        "    classname = m.__class__.__name__\n",
        "    if classname.find('Linear') != -1:\n",
        "        m.weight.data.normal_(0.0, 0.02)\n",
        "        m.bias.data.fill_(0)\n",
        "\n",
        "\n",
        "class Discriminator(nn.Module):\n",
        "    def __init__(self, opt):\n",
        "        super(Discriminator, self).__init__()\n",
        "        self.fc1 = nn.Linear(opt.res_size + opt.att_num, 1024)\n",
        "        self.attn = AxialAttention(\n",
        "            dim = 1024,\n",
        "            heads = 8,\n",
        "            dim_index = 1,\n",
        "            num_dimensions = 1\n",
        "        )\n",
        "        self.fc2 = nn.Linear(1024, 1)\n",
        "        self.lrelu = nn.LeakyReLU(0.2, True)\n",
        "\n",
        "        # self.apply(weights_init)\n",
        "\n",
        "    def forward(self, x, att):\n",
        "        h = torch.cat((x, att), 1)\n",
        "        h = self.lrelu(self.fc1(h))\n",
        "        h = torch.unsqueeze(h,2)\n",
        "        h = self.attn(h)\n",
        "        h = torch.squeeze(h)\n",
        "        h = self.fc2(h)\n",
        "        return h\n",
        "\n",
        "# class Discriminator(nn.Module):\n",
        "#     def __init__(self, opt): \n",
        "#         super(Discriminator, self).__init__()\n",
        "#         self.fc1 = nn.Linear(opt.res_size + opt.att_num, 1024)\n",
        "#         self.fc2 = nn.Linear(1024, 1024)\n",
        "#         self.fc3 = nn.Linear(1024, 1024)\n",
        "#         self.fc4 = nn.Linear(1024, 1)\n",
        "#         self.lrelu = nn.LeakyReLU(0.2, True)\n",
        "#         self.apply(weights_init)\n",
        "\n",
        "#     def forward(self, x, att):\n",
        "#         h = torch.cat((x, att), 1) \n",
        "#         h = self.lrelu(self.fc1(h))\n",
        "#         h = self.lrelu(self.fc2(h))\n",
        "#         h = self.lrelu(self.fc3(h))\n",
        "#         h = self.fc4(h)\n",
        "#         return h\n",
        "\n",
        "# class Discriminator(nn.Module):\n",
        "#     def __init__(self, opt): \n",
        "#         super(Discriminator, self).__init__()\n",
        "#         self.fc1 = nn.Linear(opt.res_size + opt.att_num, 1024)\n",
        "#         self.fc2 = nn.Linear(1024, 1)\n",
        "#         self.fc_skip = nn.Linear(opt.att_num, 1024)\n",
        "#         self.lrelu = nn.LeakyReLU(0.2, True)\n",
        "#         self.sigmoid = nn.Sigmoid()\n",
        "\n",
        "#         self.apply(weights_init)\n",
        "\n",
        "#     def forward(self, x, att):\n",
        "#         h = torch.cat((x, att), 1) \n",
        "#         h = self.lrelu(self.fc1(h))\n",
        "#         h2 = self.lrelu(self.fc_skip(att))\n",
        "#         h = self.sigmoid(self.fc2(h+h2))\n",
        "#         return h\n",
        "\n",
        "\n",
        "class Generator(nn.Module):\n",
        "    def __init__(self, opt):\n",
        "        super(Generator, self).__init__()\n",
        "        self.fc1 = nn.Linear(opt.att_num + opt.nz_size, 4096)\n",
        "        self.attn = AxialAttention(\n",
        "            dim = 4096,\n",
        "            heads = 8,\n",
        "            dim_index = 1,\n",
        "            num_dimensions = 1\n",
        "        )\n",
        "        self.fc2 = nn.Linear(4096, opt.res_size)\n",
        "        self.lrelu = nn.LeakyReLU(0.2, True)\n",
        "        self.relu = nn.ReLU(True)\n",
        "\n",
        "        # self.apply(weights_init)\n",
        "\n",
        "    def forward(self, noise, att):\n",
        "        h = torch.cat((noise, att), 1)\n",
        "        h = self.lrelu(self.fc1(h))\n",
        "        h = torch.unsqueeze(h,2)\n",
        "        h = self.attn(h)\n",
        "        h = torch.squeeze(h)\n",
        "        h = self.relu(self.fc2(h))\n",
        "        return h\n",
        "\n",
        "\n",
        "# class Generator(nn.Module):\n",
        "#     def __init__(self, opt):\n",
        "#         super(Generator, self).__init__()\n",
        "#         self.fc1 = nn.Linear(opt.att_num + opt.nz_size, 4096)\n",
        "#         self.fc2 = nn.Linear(4096, 4096)\n",
        "#         self.fc3 = nn.Linear(4096, 4096)\n",
        "#         self.fc4 = nn.Linear(4096, opt.res_size)\n",
        "#         self.lrelu = nn.LeakyReLU(0.2, True)\n",
        "#         #self.prelu = nn.PReLU()\n",
        "#         self.relu = nn.ReLU(True)\n",
        "\n",
        "#         self.apply(weights_init)\n",
        "\n",
        "#     def forward(self, noise, att):\n",
        "#         h = torch.cat((noise, att), 1)\n",
        "#         h = self.lrelu(self.fc1(h))\n",
        "#         h = self.lrelu(self.fc2(h))\n",
        "#         h = self.lrelu(self.fc3(h))\n",
        "#         h = self.relu(self.fc4(h))\n",
        "#         return h\n",
        "\n",
        "# class Generator(nn.Module):\n",
        "#     def __init__(self, opt):\n",
        "#         super(Generator, self).__init__()\n",
        "#         self.fc1 = nn.Linear(opt.att_num + opt.nz_size, 4096)\n",
        "#         #self.fc2 = nn.Linear(opt.ngh, opt.ngh)\n",
        "#         #self.fc2 = nn.Linear(opt.ngh, 1024)\n",
        "#         self.fc2 = nn.Linear(4096, opt.res_size)\n",
        "#         self.fc_skip = nn.Linear(opt.att_num, opt.res_size)\n",
        "#         self.lrelu = nn.LeakyReLU(0.2, True)\n",
        "#         #self.prelu = nn.PReLU()\n",
        "#         self.relu = nn.ReLU(True)\n",
        "        \n",
        "#         self.apply(weights_init)\n",
        "\n",
        "#     def forward(self, noise, att):\n",
        "#         h = torch.cat((noise, att), 1)\n",
        "#         h = self.lrelu(self.fc1(h))\n",
        "#         #h = self.lrelu(self.fc2(h))\n",
        "#         h = self.relu(self.fc2(h))\n",
        "#         h2 = self.fc_skip(att)\n",
        "#         return h+h2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: axial_attention in /usr/local/lib/python3.7/dist-packages (0.5.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from axial_attention) (1.8.1+cu101)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->axial_attention) (3.7.4.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch->axial_attention) (1.19.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9j6ahL1kKkV"
      },
      "source": [
        "import torch\n",
        "# import GAN\n",
        "import torch.optim as optim\n",
        "from torch.utils.data.dataloader import DataLoader\n",
        "import scipy.io as sio\n",
        "# import train_classifier\n",
        "\n",
        "\n",
        "class init_models:\n",
        "    def __init__(self, opt):\n",
        "        self.cls = classifier(opt.res_size, opt.class_num)\n",
        "        self.netD = Discriminator(opt=opt)\n",
        "        self.netG = Generator(opt=opt)\n",
        "        self.optimizerD = optim.Adam(self.netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))\n",
        "        self.optimizerG = optim.Adam(self.netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))\n",
        "        self.optimizerC = optim.Adam(self.cls.parameters(), lr=opt.lr_c, betas=(opt.beta1, 0.999))\n",
        "        self.cls_criterion = torch.nn.NLLLoss()\n",
        "        if opt.cuda:\n",
        "            self.cls.cuda()\n",
        "            self.netD.cuda()\n",
        "            self.netG.cuda()\n",
        "            self.cls_criterion.cuda()\n",
        "\n",
        "\n",
        "class images_set_train:\n",
        "    def __init__(self, opt):\n",
        "        att_origin = sio.loadmat(opt.att_path)\n",
        "        res_origin = sio.loadmat(opt.res_path)\n",
        "        att = att_origin['att']\n",
        "        res = res_origin['features']\n",
        "        label = res_origin['labels'] - 1\n",
        "        loc = att_origin['trainval_loc'].squeeze() - 1\n",
        "\n",
        "        self.label = torch.from_numpy(label[loc]).long()\n",
        "        self.res = torch.from_numpy(res[:, loc]).float().T\n",
        "        F.normalize(self.res,p=2,dim=1)\n",
        "        self.att = torch.from_numpy(att).float().T\n",
        "        F.normalize(self.att,p=2,dim=1)\n",
        "        opt.res_size = self.res.shape[1]\n",
        "        opt.att_num = self.att.shape[1]\n",
        "        opt.class_num = self.att.shape[0]\n",
        "        opt.nz_size = int(opt.res_size * opt.nz_res_ratio)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        \"\"\"\n",
        "        :param index: the index of the res feature.\n",
        "        :return: res, label, att.\n",
        "        \"\"\"\n",
        "        return self.res[index, :], self.label[index], self.att[self.label[index][0]]\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.label.shape[0]\n",
        "\n",
        "\n",
        "class images_set_test:\n",
        "    def __init__(self, opt):\n",
        "        att_origin = sio.loadmat(opt.att_path)\n",
        "        res_origin = sio.loadmat(opt.res_path)\n",
        "        att = att_origin['att']\n",
        "        res = res_origin['features']\n",
        "        label = res_origin['labels'] - 1\n",
        "        loc = att_origin['test_unseen_loc'].squeeze() - 1\n",
        "\n",
        "        self.label = torch.from_numpy(label[loc]).long()\n",
        "        self.res = torch.from_numpy(res[:, loc]).float().T\n",
        "        F.normalize(self.res,p=2,dim=1)\n",
        "        self.att = torch.from_numpy(att).float().T\n",
        "        F.normalize(self.att,p=2,dim=1)\n",
        "\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        \"\"\"\n",
        "        :param index: the index of the res feature.\n",
        "        :return: res, label, att.\n",
        "        \"\"\"\n",
        "        return self.res[index, :], self.label[index], self.att[self.label[index][0]]\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.label.shape[0]\n",
        "\n",
        "\n",
        "class loaders:\n",
        "    def __init__(self, opt):\n",
        "        img_data_train = images_set_train(opt)\n",
        "        img_data_test = images_set_test(opt)\n",
        "        self.img_loader_train = DataLoader(img_data_train, batch_size=opt.batch_size, shuffle=opt.shuffle)\n",
        "        self.img_loader_test = DataLoader(img_data_test, batch_size=opt.batch_size, shuffle=opt.shuffle)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zxoDR-Z2lzSh",
        "outputId": "12324aa5-e36a-4528-a6e4-14abaef6d03e"
      },
      "source": [
        "print('Preparing Args')\n",
        "\n",
        "opt = init_args()\n",
        "\n",
        "print('Preparing Loaders')\n",
        "\n",
        "loaders = loaders(opt=opt)\n",
        "\n",
        "print('Preparing Models')\n",
        "\n",
        "models = init_models(opt=opt)\n",
        "\n",
        "print('Start Pre-Training')\n",
        "\n",
        "pre_train(opt=opt, models=models, loaders=loaders)\n",
        "\n",
        "print('Start First Training Stage')\n",
        "\n",
        "first_stage_train(opt=opt, models=models, loaders=loaders)\n",
        "\n",
        "second_stage_train(opt=opt, models=models, loaders=loaders)\n",
        "\n",
        "final_test(opt=opt, models=models, loaders=loaders)\n",
        "\n",
        "print('Done!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Preparing Args\n",
            "Preparing Loaders\n",
            "Preparing Models\n",
            "Start Pre-Training\n",
            "Pre-Training Acc = 0.5164380798709157\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.8622932634126664\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9009681323114159\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9181625655506253\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9280455828963292\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9358612343686971\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9420129084308189\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9477107704719645\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9522993142396128\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9561819281968535\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9584005647438483\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9631403791851553\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9653085921742638\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.967628075837031\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9712585720048407\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9732755143202905\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9755445744251714\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.97650262202501\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9787716821298911\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9802339653085922\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9822004840661557\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9834610730133119\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9848729326341267\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9849737797498992\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.986789027833804\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9875958047599839\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9887555465913674\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9899152884227511\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.990520371117386\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9915792658329972\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9921843485276322\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9926381605486083\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9937474788221057\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9941508672851956\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.994856797095603\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9950584913271481\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.995260185558693\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9961173860427591\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9957644211375555\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9964199273900767\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9967224687373941\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9971258572004841\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9974283985478015\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9972771278741428\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9975796692214602\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9979830576845502\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9976805163372328\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9979830576845502\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.998336022589754\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9983864461476402\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9984368697055265\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9986385639370714\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9986385639370714\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9987898346107301\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9990419524001614\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9987898346107301\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9989915288422752\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9992436466317063\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9991427995159339\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9992436466317063\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9992436466317063\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9991932230738201\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9993949173053651\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9994453408632513\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9995461879790238\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9994453408632513\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9995461879790238\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9994957644211375\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9995461879790238\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.99959661153691\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9996470350947962\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9996974586526826\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9996974586526826\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9996470350947962\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997478822105688\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9996974586526826\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9996470350947962\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9997983057684551\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Pre-Training Acc = 0.9998487293263413\n",
            "-----------------------------------------------------\n",
            "Final Pre-Training Acc = 0.9998487293263413\n",
            "Start First Training Stage\n",
            "The 1th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 2th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 3th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 4th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 5th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 6th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 7th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 8th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 9th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 10th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 11th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 12th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 13th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 14th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 15th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 16th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 17th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 18th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 19th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 20th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 21th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 22th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 23th epoch starts.\n",
            "This epoch use 0 mins 3 secs\n",
            "The 24th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 25th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 26th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 27th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 28th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 29th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 30th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 31th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 32th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 33th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 34th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 35th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 36th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 37th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 38th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 39th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 40th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 41th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 42th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 43th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 44th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 45th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 46th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 47th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 48th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 49th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 50th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 51th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 52th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 53th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 54th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 55th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 56th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 57th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 58th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 59th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 60th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 61th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 62th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 63th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 64th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 65th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 66th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 67th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 68th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 69th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 70th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 71th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 72th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 73th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 74th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 75th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 76th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 77th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 78th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 79th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 80th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 81th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 82th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 83th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 84th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 85th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 86th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 87th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 88th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 89th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 90th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 91th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 92th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 93th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 94th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 95th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 96th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 97th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 98th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 99th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 100th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 101th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 102th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 103th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 104th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 105th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 106th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 107th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 108th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 109th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 110th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 111th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 112th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 113th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 114th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 115th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 116th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 117th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 118th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 119th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 120th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 121th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 122th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 123th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 124th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 125th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 126th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 127th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 128th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 129th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 130th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 131th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 132th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 133th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 134th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 135th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 136th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 137th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 138th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 139th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 140th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 141th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 142th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 143th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 144th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 145th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 146th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 147th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 148th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 149th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 150th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 151th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 152th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 153th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 154th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 155th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 156th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 157th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 158th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 159th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 160th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 161th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 162th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 163th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 164th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 165th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 166th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 167th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 168th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 169th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 170th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 171th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 172th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 173th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 174th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 175th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 176th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 177th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 178th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 179th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 180th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 181th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 182th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 183th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 184th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 185th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 186th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 187th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 188th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 189th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 190th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 191th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 192th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 193th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 194th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 195th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 196th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 197th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 198th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 199th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 200th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 201th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 202th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 203th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 204th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 205th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 206th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 207th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 208th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 209th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 210th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 211th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 212th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 213th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 214th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 215th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 216th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 217th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 218th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 219th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 220th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 221th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 222th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 223th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 224th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 225th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 226th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 227th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 228th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 229th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 230th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 231th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 232th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 233th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 234th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 235th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 236th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 237th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 238th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 239th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 240th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 241th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 242th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 243th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 244th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 245th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 246th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 247th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 248th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 249th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 250th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 251th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 252th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 253th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 254th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 255th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 256th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 257th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 258th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 259th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 260th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 261th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 262th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 263th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 264th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 265th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 266th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 267th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 268th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 269th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 270th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 271th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 272th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 273th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 274th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 275th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 276th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 277th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 278th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 279th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 280th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 281th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 282th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 283th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 284th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 285th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 286th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 287th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 288th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 289th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 290th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 291th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 292th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 293th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 294th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 295th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 296th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 297th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 298th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 299th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 300th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 301th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 302th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 303th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 304th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 305th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 306th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 307th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 308th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 309th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 310th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 311th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 312th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 313th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 314th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 315th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 316th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 317th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 318th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 319th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 320th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 321th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 322th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 323th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 324th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 325th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 326th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 327th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 328th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 329th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 330th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 331th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 332th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 333th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 334th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 335th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 336th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 337th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 338th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 339th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 340th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 341th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 342th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 343th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 344th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 345th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 346th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 347th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 348th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 349th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 350th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 351th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 352th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 353th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 354th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 355th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 356th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 357th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 358th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 359th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 360th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 361th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 362th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 363th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 364th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 365th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 366th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 367th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 368th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 369th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 370th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 371th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 372th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 373th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 374th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 375th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 376th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 377th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 378th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 379th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 380th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 381th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 382th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 383th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 384th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 385th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 386th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 387th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 388th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 389th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 390th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 391th epoch starts.\n",
            "This epoch use 0 mins 7 secs\n",
            "The 392th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 393th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 394th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 395th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 396th epoch starts.\n",
            "This epoch use 0 mins 6 secs\n",
            "The 397th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 398th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 399th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 400th epoch starts.\n",
            "This epoch use 0 mins 4 secs\n",
            "The 401th epoch starts.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}