vqvae_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.6.0.dev20200528\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.datasets import MNIST,FashionMNIST\n",
    "from torchvision import transforms\n",
    "from torch.optim import Adam\n",
    "import matplotlib.pyplot as plt\n",
    "from torchvision.utils import make_grid\n",
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "from vqvae import VQVAE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mod this as needed\n",
    "batch_size=64\n",
    "num_workers=8\n",
    "##############\n",
    "\n",
    "\n",
    "transform = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "    ])\n",
    "train_dataset = FashionMNIST('DATA', download=True, train=True, transform=transform)\n",
    "\n",
    "\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True,\n",
    "        num_workers=num_workers, pin_memory=True)\n",
    "\n",
    "\n",
    "test_dataset = FashionMNIST('DATA', download=True, train=False, transform=transform)\n",
    "\n",
    "\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset,\n",
    "        batch_size=batch_size, shuffle=True,\n",
    "        num_workers=num_workers, pin_memory=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_hiddens = 128\n",
    "num_residual_hiddens = 32\n",
    "num_residual_layers = 2\n",
    "embedding_dim = 32\n",
    "num_embeddings = 128\n",
    "in_channels=1\n",
    "nepochs=50 #number of epochs to train\n",
    "lr=1e-3 #learning rate for optimizer\n",
    "show_every=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_vqvae=VQVAE(num_hiddens=num_hiddens,num_embeddings=num_embeddings,embedding_dim=embedding_dim,\n",
    "            num_residual_layers = num_residual_layers, num_residual_hiddens=num_residual_hiddens).cuda()#TODO add params...\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, it:0/938, loss:0.3331460654735565\n",
      "epoch: 0, it:100/938, loss:0.046841464936733246\n",
      "epoch: 0, it:200/938, loss:0.024449774995446205\n",
      "epoch: 0, it:300/938, loss:0.022074755281209946\n",
      "epoch: 0, it:400/938, loss:0.022053752094507217\n",
      "epoch: 0, it:500/938, loss:0.020637452602386475\n",
      "epoch: 0, it:600/938, loss:0.017913194373250008\n",
      "epoch: 0, it:700/938, loss:0.01577049493789673\n",
      "epoch: 0, it:800/938, loss:0.01718159019947052\n",
      "epoch: 0, it:900/938, loss:0.0168134905397892\n",
      "epoch: 1, it:0/938, loss:0.013052497990429401\n",
      "epoch: 1, it:100/938, loss:0.014655216597020626\n",
      "epoch: 1, it:200/938, loss:0.01137523353099823\n",
      "epoch: 1, it:300/938, loss:0.014037971384823322\n",
      "epoch: 1, it:400/938, loss:0.011562710627913475\n",
      "epoch: 1, it:500/938, loss:0.013228625990450382\n",
      "epoch: 1, it:600/938, loss:0.011797762475907803\n",
      "epoch: 1, it:700/938, loss:0.01077328808605671\n",
      "epoch: 1, it:800/938, loss:0.015478963032364845\n",
      "epoch: 1, it:900/938, loss:0.011364303529262543\n",
      "epoch: 2, it:0/938, loss:0.00910993479192257\n",
      "epoch: 2, it:100/938, loss:0.009022160433232784\n",
      "epoch: 2, it:200/938, loss:0.015101253055036068\n",
      "epoch: 2, it:300/938, loss:0.00953570194542408\n",
      "epoch: 2, it:400/938, loss:0.009612527675926685\n",
      "epoch: 2, it:500/938, loss:0.009259531274437904\n",
      "epoch: 2, it:600/938, loss:0.009157043881714344\n",
      "epoch: 2, it:700/938, loss:0.009431961923837662\n",
      "epoch: 2, it:800/938, loss:0.009124277159571648\n",
      "epoch: 2, it:900/938, loss:0.008557968772947788\n",
      "epoch: 3, it:0/938, loss:0.009163936600089073\n",
      "epoch: 3, it:100/938, loss:0.009415228851139545\n",
      "epoch: 3, it:200/938, loss:0.007839178666472435\n",
      "epoch: 3, it:300/938, loss:0.00868900865316391\n",
      "epoch: 3, it:400/938, loss:0.00949865859001875\n",
      "epoch: 3, it:500/938, loss:0.009553774259984493\n",
      "epoch: 3, it:600/938, loss:0.013918749988079071\n",
      "epoch: 3, it:700/938, loss:0.009382412768900394\n",
      "epoch: 3, it:800/938, loss:0.008256789296865463\n",
      "epoch: 3, it:900/938, loss:0.009883011691272259\n",
      "epoch: 4, it:0/938, loss:0.00911046378314495\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-18475de52c1c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mimages\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Backward & update weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_hat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel_vqvae\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/kaggle/working/vqvae.py\u001b[0m in \u001b[0;36mcompute_gradients\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0;31m#####updates only the embeddings####\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m         \u001b[0ml_commit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m         \u001b[0;31m###################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mL_likelihood\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ml_emb\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ml_commit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    182\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m         \"\"\"\n\u001b[0;32m--> 184\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    123\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    124\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "optimizer = Adam(model_vqvae.parameters(), lr=lr, betas=(0, 0.99))\n",
    "criteria=nn.CrossEntropyLoss()\n",
    "\n",
    "for epoch in range(nepochs):\n",
    "    for it,(images, labels) in enumerate(train_loader):\n",
    "        images=images.cuda()\n",
    "        optimizer.zero_grad() # Backward & update weights\n",
    "        loss, x_hat=model_vqvae.compute_gradients(images)\n",
    "        optimizer.step()\n",
    "        \n",
    "        if it%show_every==0:\n",
    "            print(f'epoch: {epoch}, it:{it}/{len(train_loader)}, loss:{loss.item()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 32])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16384"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAT6klEQVR4nO3dX2xV15UG8G+BIQm2wTiYYMCaEoKSoJECIwtNlNEoo2iqlBfSSB2Vh4qRoroPrdRKfWiUeWheRopG03b6MKpEJ6h01ElVqY3CQzQqIpWiRkoVgxhDQgJMQoBibAeC//A34DUPvowc4rO+m7Pvvee2+/tJyOYu73u2z73L19frrL3N3SEif/4WVT0BEWkNJbtIJpTsIplQsotkQskukomOVh5s0aJFvmhRNT9fzCyMt3NVorOzM4zffffdhbEbN26EYz/55JMwzh6vJUuWhPFly5YVxmZnZ8Ox09PTYfzq1athPLr/Zj8Pq3o+zc7OYnZ2dsEne1Kym9mTAH4MYDGA/3D3F6KvX7RoEVasWFH6eNEJZA8ei7MnfXTsxYsXh2PZD5pbt26F8UcffTSMP/TQQ4Wx06dPh2NHR0fD+F133RXG161bF8a3bt1aGLty5Uo49rXXXgvjR44cCePRD7roBySQ/pixeHT/KS9Mk5OThbHSP97MbDGAfwfwJQCbAew0s81l709Emivld5ltAE66+/vufgPALwHsaMy0RKTRUpJ9HYAz8/5/tnbbp5jZkJkNm9lwO78vFvlzl5LsC72x+Ew2u/tudx9090H2XkREmicl2c8CGJj3//UAzqVNR0SaJSXZ3wKwycw2mNlSAF8FsK8x0xKRRrOU99Fmth3Av2Gu9LbH3f85+vqOjg7v6ekpjLO6a1Q+Y99Hap19+fLlhbFNmzaFYx988MEwvnbt2jDOSnsrV64sjEWlLwBYv359GF+zZk0Y/+CDD8L4yZMnC2NvvPFGOPbSpUth/PLly2H8+PHjhbF33nknHMvOOSvlsmsAbt68WRhj1y5Ez+XJyUncvHmz8XV2d38VwKsp9yEiraHLZUUyoWQXyYSSXSQTSnaRTCjZRTKhZBfJRFKd/fNasmSJRzVh1nu9dOnSwhire167di2Ms2M//fTThbEHHnggHDs+Ph7G2dxYm+nY2FhhjNWLH3744TDOrgFg9er33nuvMMbq5KzG39ERV477+voKYyMjI+HYDz/8MIzPzMyEcYa1wEaiOvvU1FRhnV2v7CKZULKLZELJLpIJJbtIJpTsIplQsotkoqVLSbt70vK+0VhWOmMtrvfcc08YHxgYKIydOnUqHBu1MwK8tZethBphJaLh4eEwzubOymdR2XDVqlXhWFY2ZG2k0YrBGzduDMd+9NFHSXH2fIrOKyu1li3b6ZVdJBNKdpFMKNlFMqFkF8mEkl0kE0p2kUwo2UUy0dI6OxAv2ZzS9peqt7c3jEfL+0bbEgO8js5q2WyH2Whu7JyyFmfWRpqyey679iHakbSeY0etw+zahWjpcICft6gdG4gfl5QdYCN6ZRfJhJJdJBNKdpFMKNlFMqFkF8mEkl0kE0p2kUy0VZ2d1Q+j/mZW92RxtnVxVDdlfddTU1Nh/Pr162G8u7s7jLMtflOOzc4bq3V3dnYWxlgvfMp9A/EaB+y+WT86k7K+Aru2oWwOJSW7mZ0CMA3gFoCb7j6Ycn8i0jyNeGX/O3ePl+0QkcrpPbtIJlKT3QH81swOmtnQQl9gZkNmNmxmw63cakpEPi311/jH3P2cma0GsN/M3nX31+d/gbvvBrAbADo6OpTtIhVJemV393O1j+MAXgawrRGTEpHGK53sZtZpZt23PwfwRQBHGzUxEWmslF/j7wPwcq2u1wHgv9z9v9mgqA7I+r6jPl/29wDWc75ixYowHtVN2diJiYkwnlJXBeJ6NbsGgPVds7ldunQpjEfHZ8dm112w9dWjY7M1Ahj2fGJzj65vYNdNRNcIRM+V0snu7u8DeKTseBFpLZXeRDKhZBfJhJJdJBNKdpFMKNlFMtHyFteU1r5oLCsxsZZG1kYatVP29fWFY0dGRsI4mxsr7UXlFnZeWAmKLXPNlmSOymOsDZQtqbx27dow3tPTUxg7ceJEODa1tZedN/Zcj5S97Fyv7CKZULKLZELJLpIJJbtIJpTsIplQsotkQskukomW19lTRHVX1lLI4tH2vkBcTx4YGAjHpi4Vzebe1dVVGGNtwyzOarpsblG7JmtxvXLlShjfsGFDGI+kbpPNxpfdVrkerMZfOK7B8xCRNqVkF8mEkl0kE0p2kUwo2UUyoWQXyYSSXSQTf1JbNkc1W9YbzeqibPneaG5se19Ww2fHZnOPetZZrZr1VafWk6NlrtlW1h9//HEYX7NmTRiPauXs+2aPKRvPHvOUJbbLLoOtV3aRTCjZRTKhZBfJhJJdJBNKdpFMKNlFMqFkF8lEW9XZWW91hNU9WS17ZmYmjEc9xOvXrw/HsrXV2fro7BqCqJbO7jv1vLF+92g8Oy+s3rxq1aowHj2m7LxEawQAad83wB/TSNP62c1sj5mNm9nRebf1mtl+MztR+7iy1NFFpGXq+RHxMwBP3nHbswAOuPsmAAdq/xeRNkaT3d1fB3Dxjpt3ANhb+3wvgKcaPC8RabCy79nvc/dRAHD3UTNbXfSFZjYEYAgo/15DRNI1Pfvcfbe7D7r7YDMX4RORWNlkHzOzfgCofRxv3JREpBnKJvs+ALtqn+8C8EpjpiMizULfs5vZSwAeB7DKzM4C+D6AFwD8ysyeAXAawFfqPWD0vj2lzs7qptH+6gDfxzzqrWb7p1+8eOffNz+tt7c3jF+9ejWMR2+Pov3RAV4vZuNZvTh6vFP7+Nm1EdHcWb8562dn1wCwxyz63tnb3bJ5QpPd3XcWhJ4odUQRqYT+PC6SCSW7SCaU7CKZULKLZELJLpKJtmpxZeWvCCtHsNIbK5VMTEwUxlirJjs2K3+xMlB03lj5im0nnbqlc9RCm7rN9oULF8L46tWFV3EnlXkBPjd2aXi0HHTKOQ3nVGqUiPzJUbKLZELJLpIJJbtIJpTsIplQsotkQskukom2qrOnbNnMavT33ntvPLEErJbd19cXxs+fPx/GU1oeWc025dqGekQtsKzFlV2/wFqHu7u7C2PsMYvGAsCyZcvC+OTkZBhPaXFlj2kRvbKLZELJLpIJJbtIJpTsIplQsotkQskukgklu0gmWl5nj2qIrMc46hFmtUdWu2Q941FdNmW753qkjGdLbLMlkVMek3riETa3aHlvIN5Km9X42fUH7BoANj46L+wagLJrBOiVXSQTSnaRTCjZRTKhZBfJhJJdJBNKdpFMKNlFMtFWdXZWm4zWdmd1U1bvvXz5chi/dOlSYYzVRVk9mNX4Wa08+t5Se6NZnZ1t2RzVylNr2dFa/kD8nOjq6grHsu8rWpMeAM6cORPGI2yb7LJr3tNXdjPbY2bjZnZ03m3Pm9kfzexw7d/2UkcXkZap59f4nwF4coHbf+TuW2r/Xm3stESk0Wiyu/vrAOL1f0Sk7aX8ge5bZjZS+zV/ZdEXmdmQmQ2b2XDZtbNEJF3ZZP8JgI0AtgAYBfCDoi90993uPujug+yPRSLSPKWS3d3H3P2Wu88C+CmAbY2dlog0WqlkN7P+ef/9MoCjRV8rIu2B1tnN7CUAjwNYZWZnAXwfwONmtgWAAzgF4Bv1HMzMwrpvyp7XbCyr2U5NTYXx3t7ewtjY2Fg4dnp6Ooz39/eH8Wgvb4BfYxBJ7VdnPefR3FNq9ABfNz7q++7s7AzHjo+Ph/GUtfzZeHZeyq7rQJPd3XcucPOLbJyItBddLiuSCSW7SCaU7CKZULKLZELJLpKJlra4ujtt14xEpRh2KS7bsjml9MaWkr5y5UoYZ2UgVlqLzilrI01pnwV4e2/KfbPyFitpRi3RrEX13XffDeOsLTmljMxaXMvmkF7ZRTKhZBfJhJJdJBNKdpFMKNlFMqFkF8mEkl0kE221lDSTsv0vq8OzFtioXfLgwYPhWFbDZ8dmou8tpd5bD1YTvn79emGMtYEuX748jJ8/fz6MR98bu7bh2rVrYZzNjd1/tHQ5uzai7PJuemUXyYSSXSQTSnaRTCjZRTKhZBfJhJJdJBNKdpFMtLTObmZh7ZPVXaNlidmywwzrOe/p6SmMnT59Ohy7bNmyMM56o9mWz1EtPbo+gI2tJ856q6PHJWWbbIDXwqPrG6L1CQD+faU8V4H4cUl5TKKYXtlFMqFkF8mEkl0kE0p2kUwo2UUyoWQXyYSSXSQTLV83PurFZX26UW2S1T1ZnPVlR/3Lx48fD8eyOjurhbO12aPx7Jyymi6bGxPVq9k5Z9dOsDp8tPUxq4Oz5wurw7O1/qNrBNiWzWUfb/rKbmYDZvY7MztmZm+b2bdrt/ea2X4zO1H7uJLdl4hUp55f428C+K67PwzgrwF808w2A3gWwAF33wTgQO3/ItKmaLK7+6i7H6p9Pg3gGIB1AHYA2Fv7sr0AnmrWJEUk3ed6Q2ZmXwCwFcAfANzn7qPA3A8EM1tw8ywzGwIwBKSvdyYi5dWdfWbWBeDXAL7j7vEKivO4+253H3T3wZTFJkUkTV3JbmZLMJfov3D339RuHjOz/lq8H8B4c6YoIo1Af423uZfjFwEcc/cfzgvtA7ALwAu1j6/Uc8CoNMDKHWFZIfEtAvutIyoTjY2NhWNZCYmV1th5icazZarZsVmclaCiEhebG2uBZUsuT0xMFMbuv//+cGzK1uIA/96iJbZZSZI9H4rU8579MQBfA3DEzA7XbnsOc0n+KzN7BsBpAF8pNQMRaQma7O7+ewBFL3tPNHY6ItIs+vO4SCaU7CKZULKLZELJLpIJJbtIJlq+lHSz2lRZyyKrw7M21KjefOHChXDs5s2bwzibewp236ydktWyWR0+Oj6rZbMltll8cnKyMMaW52Z1cnZeojo6EJ93dt9l6ZVdJBNKdpFMKNlFMqFkF8mEkl0kE0p2kUwo2UUy0fKlpKPaKqvZRvXHzs7OcCyre7Itm6Px09PT4VhW409dljg6LytXxov+smOzmi87b9H9sz5/VuseGBgI49FW2k88ETdssmOzx5Qt4R1dI5CyfkFEr+wimVCyi2RCyS6SCSW7SCaU7CKZULKLZELJLpKJltbZgbj+yGq6US98tAUuwOvJ3d3dYTyqfbJ1vtma9KxuevHixTAenTf2fbN6ceq68tF5Y9tBs/jo6GgYj3rGp6biTY2iLboBvqY9O6/ReSm7LjyjV3aRTCjZRTKhZBfJhJJdJBNKdpFMKNlFMqFkF8lEPfuzDwD4OYA1AGYB7Hb3H5vZ8wC+DuD2JtjPufur7P6iGiKrs0c1Y1aTZf3FrB8+qpuy+z58+HAYZ2u3s372mZmZwlhXV1c4lvX5s2Ozdemjc8OuT2C1ahZn6/lHHnnkkTB+6NChMJ7S796sdePruajmJoDvuvshM+sGcNDM9tdiP3L3f23KzESkoerZn30UwGjt82kzOwZgXbMnJiKN9bnes5vZFwBsBfCH2k3fMrMRM9tjZguuf2RmQ2Y2bGbDzboMUES4upPdzLoA/BrAd9x9CsBPAGwEsAVzr/w/WGicu+9290F3H2TvY0SkeerKPjNbgrlE/4W7/wYA3H3M3W+5+yyAnwLY1rxpikgqmuw217L1IoBj7v7Debf3z/uyLwM42vjpiUij1PPX+McAfA3AETO7XUN6DsBOM9sCwAGcAvCNeg6YUsKK3vOzZYnZ3wtYi2tU9mPbPbMWVdYuyeYWlc/YOWWlM/a9sfHReWfttyzOSpbReb169Wo4NuWcA7zlOmXLZtYyXaSev8b/HsBC905r6iLSPvQXM5FMKNlFMqFkF8mEkl0kE0p2kUwo2UUy0fKlpCOsfhjVH1nN9dy5c6XvGwAOHjxYGItaTOvB2lDZZcbR+NTtoFO3dI4eU/aYRdsaA/zaiRUrVhTG3nzzzXBsX19fGGd1dLbEdsql42V7TPTKLpIJJbtIJpTsIplQsotkQskukgklu0gmlOwimTDW79zQg5lNAPhw3k2rAHzUsgl8Pu06t3adF6C5ldXIuf2Fuy94kUBLk/0zBzcbdvfByiYQaNe5teu8AM2trFbNTb/Gi2RCyS6SiaqTfXfFx4+069zadV6A5lZWS+ZW6Xt2EWmdql/ZRaRFlOwimagk2c3sSTN7z8xOmtmzVcyhiJmdMrMjZnbYzIYrnsseMxs3s6Pzbus1s/1mdqL2ccE99iqa2/Nm9sfauTtsZtsrmtuAmf3OzI6Z2dtm9u3a7ZWeu2BeLTlvLX/PbmaLARwH8PcAzgJ4C8BOd3+npRMpYGanAAy6e+UXYJjZ3wKYAfBzd//L2m3/AuCiu79Q+0G50t2/1yZzex7ATNXbeNd2K+qfv804gKcA/CMqPHfBvP4BLThvVbyybwNw0t3fd/cbAH4JYEcF82h77v46gDu3k9kBYG/t872Ye7K0XMHc2oK7j7r7odrn0wBubzNe6bkL5tUSVST7OgBn5v3/LNprv3cH8FszO2hmQ1VPZgH3ufsoMPfkAbC64vnciW7j3Up3bDPeNueuzPbnqapI9oUWJWun+t9j7v5XAL4E4Ju1X1elPnVt490qC2wz3hbKbn+eqopkPwtgYN7/1wOIV4NsIXc/V/s4DuBltN9W1GO3d9CtfRyveD7/r5228V5om3G0wbmrcvvzKpL9LQCbzGyDmS0F8FUA+yqYx2eYWWftDycws04AX0T7bUW9D8Cu2ue7ALxS4Vw+pV228S7aZhwVn7vKtz9395b/A7Adc3+R/18A/1TFHArmdT+A/6n9e7vquQF4CXO/1n2Cud+IngFwL4ADAE7UPva20dz+E8ARACOYS6z+iub2N5h7azgC4HDt3/aqz10wr5acN10uK5IJXUEnkgklu0gmlOwimVCyi2RCyS6SCSW7SCaU7CKZ+D+7RqKYw/lSjAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(model_vqvae.codebook.shape)\n",
    "shape=model_vqvae.ze.shape\n",
    "indices=model_vqvae.indices\n",
    "indices=indices.reshape(shape[0],shape[2], shape[3])\n",
    "\n",
    "out=model_vqvae.compute_out_from_indices(indices)\n",
    "\n",
    "# print(model.indices.shape)#batchx16x16\n",
    "# zq=model.codebook[model.indices]\n",
    "# shp=model.ze.shape\n",
    "# zq=zq.reshape(shp[0],shp[2],shp[3],-1)\n",
    "# zq=zq.permute(0,3,1,2)\n",
    "# print(zq.shape)\n",
    "# out=model.decoder(zq)\n",
    "plt.imshow(out[0,0].detach().cpu(),cmap='gray')\n",
    "64*16*16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Reconstructed from the sampled codes')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAEtCAYAAADHtl7HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3Scd33n8c83tiTLkmzL1/hGTBKTkJAbaxJupw2loSnQJWWb0rTLJi3FcBZOyx62lGbbbdrSLu0pELYXekKTJmxLCj0UyLbdBYpL2UCuNsEJxHFuvsWyZFuyLdmy5ctv/5hHzViR9P3+NM9IM/b7dY6PpZmvnuc3z+U3X808+oyllAQAAIC4c2Z6AAAAAM2GBgoAACATDRQAAEAmGigAAIBMNFAAAACZaKAAAAAy0UAhzMx+YGbXzvQ4AJwdzOxbZvbLE9xnZvZXZjZgZg9P99jGY2bbzOzHZ3ocZZls+9fzZzPWcZuZ/XU91zEZGiiEpZQuTSl9q57rMLO7zexj9VwHUE/Fk+iwmQ2Z2Z7imO6c6XGNx8ySmV1Yp2WvKZY/ux7Ll/RGSddJWpVSurpO65gQcxVooACgfD+VUuqUdKWkqyT9xgyPZ0rq2PyU4TxJ21JKh8e7s8HHjjMADRTCRl+eLl42/aKZfc7MBou39taNqfsNM/th8fL6X5nZnOK+W8zs/jHLTWZ2oZmtl/QLkj5S/Pb+v6f3EQLlSintkfQ1VRopSZKZvdbMvmtmB8zs+9Vvi5vZwuJ82V2cO1+puu+9ZvaMmfWb2X1mtqLqvmRm7zezp4uf+zMzs+K+C83sX83soJntM7MvFLd/u/jx7xfn27vM7Foz22Vmv25meyT91WTnbPF1u5l9wsy2F+u438zaJY0u/0Cx/NcV9b9kZk8W4/yamZ1XtdzrzGxLsZw/lWTjbVcze4+kv5T0umLZvzPB2NvM7PZie+4uvm4rljFa/xEz6zOzHjO7wczeamZbi+186wTrn2yuutLMNheP4Qujc1/xc283s8eKff9dM7t8guWbmX2qGNfBYnmvKu57m5l9z8wOmdlOM7ut6udGX/X7xeK+geK4eE2xjAPFdh2tv8XMvmNmf1KsZ4uZvXm8MZW174raWWZ2q5k9a5XnkI1mtrq47/Vm9kixnEfM7PVVP/fy4lgeNLNvSFo8ZrmTnVu3mNlzxc8+b2a/MNH4wlJK/ONf6J+kbZJ+XNJtko5KequkWZL+h6QHx9Q9IWm1pIWSviPpY8V9t0i6f8xyk6QLi6/vHq3lH/+a8d/oeVJ8vUrS45I+XXy/UtL+4tw5R5W3oPZLWlLc/4+SviCpW1KLpB8tbv8xSfskvVpSm6Q/kfTtqnUmSf8gaYGkl0naK+n64r57Jf23Yn1zJL1xzM9dWPX9tZJOSPrDYj3tgXP2zyR9q3hssyS9vvjZNUXd7Kqfu0HSM5JeKWm2pN+U9N3ivsWSDkn6meKx/5diLL88wXY+bVwTjP13JT0oaamkJZK+K+n3xtT/92J97y222+cldUm6VJV57vwJ1v+SuarY9w9LWqHK3PekpPcX971aUp+ka4rtdHNR3zbOsn9C0sZif1qxvZZXjfuyYn9eLqlX0g3FfaPb/C+Kff2W4jF8pdgGK4sx/GjVNjxRbOsWSe+SdFDSwuL+b41u/5L33a+pcl5cVDy+KyQtKrbZgKR3F+u4qfh+UfFzD0j6ZLF/f0TSoKS/9s4tSR3F+C4qapdLurTmc32mJxv+Nc8/nd5A/XPV7ZdIGh5T9/6q798q6dni61tEA8W/M/hfcfwPFZN7kvRNSQuK+35d0v8aU/81VZ5Ml0s6Jal7nGXeKemPqr7vlHRc0pri+6TTG6MvSvpo8fXnJN2hyrVCY5c7XgM1ImlO1W0TnrPFE9WwpCvGWfYavbSB+j+S3lP1/TmSjqjydtx/0um/iJmkXcproMaO/VlJb636/idUedtvtH5Y0qzi+65ivNdU1W9U0ZyMs/6XzFXFvv+PVd//kaS/KL7+jIrmrer+p1Q0M2Nu/zFJWyW9VtI5zvF2u6RPjdnmK6vu3y/pXVXff0nSh6q24W5JVnX/w5LeXXz9Lb3YQJW5756S9I5xbn+3pIfH3PZAMc6XqdKUdVTd93m92EBNdm51SDog6T9Iai/rXOctPEzVnqqvj0iaY6dfc7Cz6uvtqvxGBpwtbkgpdanyJH2xXnyr4TxJNxZvMRwwswOqXAy9XJVXbPtTSgPjLG+FKueRJCmlNKTKE+PKqpqx5+TohesfUeUJ7WGrvN3+S87Y96aUjgYeo4rHNUeVRiXiPEmfrnrs/cXYVqryGP9t3kiVZ8Cd4y5lYmPHftp200vnov0ppZPF18PF/71V9w/rxe0YNdF+OE/Sh8fs+9UaZ25MKW2Q9KeqvLrXa2Z3mNk8STKza8zsX8xsr5kdlPR+jXkra5zHMNljeqHY1qMmmq/L3HerNf4xM3Z/jY5ndB0D6fRr3qprJzy3ip95lyrbqsfM/tHMLp5kfCE0UKiX1VVfv0yV33Ik6bCkuaN3mNm5Y34uCThDpJT+VZVXKv64uGmnKr8lL6j615FS+nhx30IzWzDOonar8gQhSTKzDlXe8nghMIY9KaX3ppRWSHqfpD+3yf/ybuw5ONk5u0+Vt4guCCxHqjzG9415/O0ppe9K6lHVvGFmptPnkYix6zxtu+n0uahWuXPVTkm/P+axz00p3TvuwlP6nymlf6fKW4mvUOVtL6nyqst9klanlOar8nbdhNcbBawstvWoibZRmftup8Y/Zsbur9HxvFCso7s49qvvq17mROeWUkpfSyldp8ovK1skfXaS8YXQQKFePmBmq8xsoaRbVbmuQ5K+L+lSM7uyuLjytjE/1yvp/OkbJlB3t0u6zsyulPTXkn7KzH6iuJB2jlUuZl6VUupR5W2SPzezbjNrMbMfKZbxeUm/WJw3bZL+QNJDKaVt3srN7EYzW1V8O6DKE//oqy6R823CczaldErSXZI+aWYrisf0umKMe1V5S7J6+X8h6TfM7NJibPPN7Mbivn8s1vPO4tXsX5E09hesXPdK+k0zW2Jmi1W53qms3KDcueqzkt5fvIJkZtZhlQvCu8YWFhd9X2NmLao0sEf14j7rUuWVyqNmdrWkn6/xcSyV9CvF8XajKtc4/dM4dWXuu7+U9HtmtrbYFpeb2aJiva8ws583s9lm9i5VLhH5h5TSdkmPSvodM2s1szdK+qmqZU54bpnZMjP790XzdUyVt9hPqkY0UKiXz0v6uqTnin8fk6SU0lZVLuz8Z0lPS7p/zM/dKemS4iXYrwhocimlvapch/RbKaWdkt6hyi8Ve1X5rfnX9OJc/G5Vrm3aosrFvh8qlvFNSb+lyvUrPar89v5zwSG8RtJDZjakyisXv5pSer647zZJ9xTn289OMH7vnP2vqlwQ/Igqb+v8oSrX7RyR9PuSvlMs/7UppS8X9/+tmR1S5Y9NfrJYzz5JN0r6uCpvT65V5Q9QavExVZ50Nxdj3FTcVoasuSql9KgqF6r/qSqN7DOqXNsznnmqNFwDqrxNtV8vvor5nyX9rpkNqtIQfrGGxyBJD6myrfepsr9+JqW0f5zxl7nvPlmM++uqXNx9pyrXJu2X9HZJHy6W8xFJby+WL1WaxWtUOc5+W5XzanR8k51b5xTL3F387I+qsh1rYqe/9QnUzsy2qXLx4D/P9FgAAOMzs1tUmavfONNjaUa8AgUAAJCJBgoAACATb+EBAABk4hUoAACATDRQAAAAmab106rNjPcLS3B65tn4Wltb3ZolS5a4NUeOHHFrjh075tZEzJ07161pb293aw4fHvfD2U+zf/9L/koX9bMvpeQfbA3unHPOSeec0zi/c0bmgWa9RKOjo8OtmTNnjlszMjLi1hw/ftytiez3lpYWtyYyx506dcqtGRwcdGuGh4fdmsi6pvuYb7Rj9uTJkxPOXzU1UGZ2vaRPq/LBiH85mvh5JmnESSpyoq5Zs8atWb9+vVuzadMmt+b55593ayKuuOIKt+byy8f98PLTPPzww27N3Xff7dZM935txGOtJGM/mqFh5Mxh55xzjubPn1/zOiP7MPKkFamJNAeR8cyaNcutkWLH8MmTfn7h6173Orfm4ov9T+LYsWOHW9PT0+PWtLW1uTUrV650a6666iq3JvJL64YNG9yaxx9/3K2JNJiRRlUqb99HaiLrKms+7e/vn3D+mnJraWazVPmcnp9UJSn0JjO7ZKrLA4DpxBwGoBa1vDZ3taRnUkrPpZRGJP2tKimgANAMmMMATFktDdRKnf5py7t0+ieDA0AjYw4DMGW1XAM13huML3lD0czWS/IvtgGA6eXOYdXzVyNdQA5g5tXSQO2StLrq+1WqfFDfaVJKd0i6Q+Kv8AA0FHcOq56/Zs+ezfwF4N/U8ivVI5LWmtnLzaxVlU8Gv6+cYQFA3TGHAZiyKb8ClVI6YWYflPQ1Vf4E+K6U0g9KGxkA1BFzGIBaTOtn4Z3Nb+G9/vWvD9W95jWvcWsiWTQ7d+50a972tre5Na985SvdmkhuR2dnp1uzePFit+bEiRNuze233+7WRLKrIut65JFH3JpnnnnGrTnDbUwprZvpQdRq9uzZacGCBZPWlBVMGJmXy8q5mTdvnluzdu1at0aSLrroIrdmxYoVbk0kd6q7u9utieQurVq1yq0599xz3ZrInBKZC77zne+4NQcOHHBrIoHCW7dudWt++MMfujVSbJ9Fjv1IAGhkbo7kJUbOof7+/gnnL66KBAAAyEQDBQAAkIkGCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGQiSLMEH/jAB9yapUuXhpa1ZcsWtyYSkBYJNduxY4db4wUHStKrX/1qtyYSRPf973/frXnooYfcmkiA2rJly9yaSPjnwoUL3ZqhoSG3RpL+7u/+LlTXhM6IIM2WlpbkhTeOjIy4y2ltbXVrIoGDR48edWsi43nnO9/p1lx44YVujST19fW5NZFxt7W1uTW9vb1uTWQejIQFR8I/I4GTTz31lFsTmd8j8+ns2f4HjSxZssSt2bx5s1sjSdu3b3dronOhJxLeHBEJ0hwYGCBIEwAAoCw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEAwUAAJCJBgoAACATDRQAAEAmP2nrLBcJkJs3b55b88ADD4TWF1lWJGTu1KlTbs3555/v1gwMDLg1GzZscGsOHjzo1syZM8etiQTIRZZz7NgxtyYy5hdeeMGtWbt2rVsjSRdffLFbEwlaRX2klNzzKhKAGTk3IwGYkRDA9vZ2t2b16tVuzbZt29waSTpx4oRbE3n8kXM4IhLc+Oijj7o1kccVCcCMzN2LFy92ayIBocPDw27N8ePH3ZoLLrjArZGkffv2lVITOWYj+yOyrWsN5OQVKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEAwUAAJCJBgoAACATDRQAAEAmgjQdkRCxSGBXWcFwUiz8LCISaDd37ly3JvL4V65cWcp4Io/96NGjbk1Kya2JiGyf/v7+0LIWLlxY63BQZ95xU2swX9kix1RLS4tbEznOpdg5HAlBjJznkXFH9kdkLpg923+qjISoRmoiAamRkN/IuiJzZfS5KxICHdnWra2tbk1kv0ZqItt6MrwCBQAAkIkGCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEkKYjEiAXCYZbunRpaH2Dg4NuTXt7u1szPDzs1kRCzSKBbQcOHHBrIoFlkfFEHvuCBQvcmrLCNiMBe52dnW5NdEyYWd4xETnOZ82aVfN6ojWrVq1yayLBhZExS9KhQ4fcmmPHjrk1XV1dbk0kSDMiMp7Ito4EV3Z0dLg1hw8fnrZ1jYyMlLIuKTY3R0TGFDnPInNzrYHKNTVQZrZN0qCkk5JOpJTW1TQaAJhGzGEApqqMV6DelFLaV8JyAGAmMIcByMY1UAAAAJlqbaCSpK+b2UYzW1/GgABgGjGHAZiSWt/Ce0NKabeZLZX0DTPbklL6dnVBMSkxMQFoRJPOYdXzV/RiWgBnh5pmhJTS7uL/PklflnT1ODV3pJTWcXEmgEbjzWHV81fkL38AnD2m3ECZWYeZdY1+Lektkp4oa2AAUE/MYQBqUctbeMskfbn4rWy2pM+nlP5vKaMCgPpjDgMwZVNuoFJKz0m6osSxNKQ5c+a4NZEAsUWLFoXWFwnSjASERcI9I2GbJ0+edGsiwZWRQNL+/n63JjLmgYEBt2bZsmVuTSSoL7Lvo0Ga+/bxl/TTaSpzmPc23qlTp9xlRM6pSMBf5JyaP3++WxMJLowsR5L27t3r1pQVcBgJnIwEgEaCRCNjjgQKlzWeyNvJbW1tpYzn+PHjbk1U5JiNPLZI+Glk/q71ukauigQAAMhEAwUAAJCJBgoAACATDRQAAEAmGigAAIBMNFAAAACZaKAAAAAy0UABAABkqvXDhM94kWC8SGBXNEhzy5Ytbs3q1avdmr6+PrcmEkQXCYGMBANGarq7u92aoaEhtyYSeldWgNyrXvUqtyYyZikWIIeZ5e2jyLEX2c+R4zMSAtjV1eXWdHR0uDVLlixxayRp8+bNbk1k3JHgzkjYZlnneSSYOBK6HAm3jASbRubTFStWuDWREOSnn37arZFi+yOy7yPbOnKeRUTGPBlegQIAAMhEAwUAAJCJBgoAACATDRQAAEAmGigAAIBMNFAAAACZaKAAAAAy0UABAABkooECAADIRBK5I5J4GknybW1tLWM4kqSDBw+6NZF09Je//OVuzfPPPx8akyeSLLxr1y63ZuHChW5NZJ8NDg66NZdddlkp49m6datbI0nt7e2hOjS3SIp0JK08UnP06FG3JpKgHfn0A0k6duyYWxNJR488tsinJEQ+SSJSE0msjow5Mi9HniuOHDni1kTm94hIMrhUXqL7dH4iQyQZfdKfL2kcAAAAZw0aKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEAwUAAJCJBgoAACATQZoliIRxLV68OLSsSEBaZFnd3d1uTSRwMhKOdvjwYbcmEtY3PDzs1kQsWbLErdmzZ49bc+DAAbemra3Nrenp6XFrJGnt2rWhOswcL1CxrDDFkZERtyYSShhZV2TM0ZDXSHBnZEyRxzZr1iy3JjKfRubBsgIgI3PloUOH3JqBgQG35txzz3VrIvN7ZPtIsWMksqzIMRTZ95FA0sjjnwyvQAEAAGSigQIAAMhEAwUAAJCJBgoAACATDRQAAEAmGigAAIBMNFAAAACZaKAAAAAyualWZnaXpLdL6kspvaq4baGkL0haI2mbpJ9NKfnJXg0mEoAZCf6KhN6tXLkyNKaywiRPnTrl1kTC4SI1keCzSCjl7t273ZrOzk635rzzznNrImO+/vrr3Zr58+e7NUNDQ26NFAuiixyzkX1/NilzDvOCNMva9pF5JxJIGTn2IsfUqlWr3BopFph78uRJtyYyp0ZCMiPrKmtbe8dGdDmRbRgJiYwELkeOj8g2lGJzc1nbKHJ8RESO/Ul/PlBzt6SxzyQflfTNlNJaSd8svgeARnS3mMMAlMxtoFJK35bUP+bmd0i6p/j6Hkk3lDwuACgFcxiAepjq61fLUko9klT8v7S8IQFA3TGHAahJ3T9M2MzWS1pf7/UAQNmq569ar5cAcGaZ6ozQa2bLJan4v2+iwpTSHSmldSmldVNcFwCULTSHVc9fZjatAwTQ2KbaQN0n6ebi65slfbWc4QDAtGAOA1ATt4Eys3slPSDpIjPbZWbvkfRxSdeZ2dOSriu+B4CGwxwGoB7ca6BSSjdNcNebSx4LAJSOOQxAPdT9IvJG1tbWVkrN8ePH3ZpokOaiRYvcmueff96teeqpp9yaV7ziFW7N1Vdf7db09PS4Nbt27XJrrr32WrdmYMDPa920aZNbs2PHDrfmqquucmu2b9/u1kTNmjXLrSFIc2Z527+sbR8JL+zo6HBrIsdU5JyKBMZKUn//2LSIl1q4cKFbEwkUjlyTFpm/I+GOkeVEwh0j528kSDIScBwJyYw8rkjosBQLAo4EgEb2fWQbRY6PWs9X/qwEAAAgEw0UAABAJhooAACATDRQAAAAmWigAAAAMtFAAQAAZKKBAgAAyEQDBQAAkOmsDtKMhMNFgugOHTrk1ixbtiw0posuusiteeyxx9yaN73pTW5NJLAsEsYWCYeLBITOnu0fjpEgtosvvtitiQRpPvvss27NnDlz3JrIMRTV3d3t1uzdu7e09eF0XuhiWfs6EvAXCdKMnOOR4yVynEfHFAmujIQyRrZ1JHDy2LFjbk1kf0QeV2SOiwRARmr279/v1ixdutStKTOYNzLuyPNJJLy6rP0xGV6BAgAAyEQDBQAAkIkGCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGQ6q4M0I6FvkVCvwcFBtyYasBcZUyTcMjKmSIBcZ2enWxMJq4vUREL/IkFske1z6aWXujUbNmxwa6655hq3Zs2aNW6NFBt35PggSLN+vHC+yPHZ0tLi1kTmi0g4bVki568kLVmyxK3Zs2ePWxPZjmWFW5YZdOsZGRlxayLHRyTYtL+/363p6upya6L7PrKsuXPnujUHDx50ayLbKHIMRY6PyfAKFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEAwUAAJCJBgoAACDTWR2k2d7e7tZEQtb6+vrcmueeey40pvvvv9+tWbFihVtz8uRJt+bw4cNuzbx589yaSDjckSNH3JqIyD6LBEkuW7bMrdm8ebNbEwnJPP/8890aSTpw4IBbU2vwG2rjhfNFwh0j4byR/RwJCoycL5GgxEjIqxR7bNO5nMg82Nra6taUtV/LelyRMQ8MDLg1q1atcmsioZVS7LkyEgAaWU5kO0aO69mza2uBeAUKAAAgEw0UAABAJhooAACATDRQAAAAmWigAAAAMtFAAQAAZKKBAgAAyEQDBQAAkOmsDtKMhHpFgtgiYZPPPvtsaEzPPPOMW/OWt7zFrYmEMkZq+vv73ZpIGFkk+C0S7BkJ7YyEB0ZEgtgix1BkOZI0PDzs1nR1dYWWhfrwwisjIYCR/RwJL4yECUbOqcg8ED2GI+GNkfMzMu9GHn8kbDQSWhoJ0ozMTZF5sKxAykigcOQ46+zsdGuk2ONfunSpW7Nz587Q+jxtbW1uTWS/TsY9As3sLjPrM7Mnqm67zcxeMLPHin9vrWkUAFAnzGEA6iHyFt7dkq4f5/ZPpZSuLP79U7nDAoDS3C3mMAAlcxuolNK3Jfnv4wBAA2IOA1APtVxE/kEz21y8PN5d2ogAYHowhwGYsqk2UJ+RdIGkKyX1SPrERIVmtt7MHjWzR6e4LgAoW2gOq56/IhcbAzh7TKmBSin1ppROppROSfqspKsnqb0jpbQupbRuqoMEgDJF57Dq+SvyF10Azh5TaqDMbHnVtz8t6YmJagGg0TCHAaiVG+BjZvdKulbSYjPbJem3JV1rZldKSpK2SXpfHccIAFPGHAagHtwGKqV00zg331mHsUy7skIQ586d69Z0d8euUY0E30X09fW5NZGwurJC7yJhfZFAzsj+OHr0qFsT2feRMa9cudKtiYQLStKOHTvcmsg2wunKmsPMzA1mjLzNV1YAZOQYPnTokFuzcOFCt6a3t9etkaTBwUG3Zvny5W7N8ePH3ZpICGREJEwxss8iIZmRx1VWIGdZIcgdHR1ujRR7zokc15H9EVlOZDtG9uukP1/TTwMAAJyFaKAAAAAy0UABAABkooECAADIRAMFAACQiQYKAAAgEw0UAABAJhooAACATGd1Kl9bW5tbEwnjWrZsmVuzZ8+e0JiOHDni1kTCv7q6utyaskL/IsFnkbDRyIe1RmoiYXWRxx4JBYwE0UVC76TYY4scs6iPlFIoNNYTOR4ix8KiRYvcmrKCNIeGhtwaKTZ/RYIZIyGZkX0xa9asUpYTmQcjIb8RZQWtRuav4eFht2bp0qVujSRt2bLFrYkEM5f1vBSZK2s9n3kFCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEAwUAAJDprA7SjIQgRkIiOzs73Zr9+/eHxhQJY4uMKVITCRqLhJpFRMYTWVckYDASwhcJ2IsE0UWWc8EFF7g1kvTVr37VrSFIc2aVcT5EQgAjIufCnDlz3JrIPLhx48bQmCLBnZExRUQef1mhjBGRc/PYsWNuTWSunDdvnlsTCW+OPPZI8KkkHT161K2JjDuyvsOHD7s1kbk5cgxNhlegAAAAMtFAAQAAZKKBAgAAyEQDBQAAkIkGCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJnO6iDNSIhWpKa9vd2t6e3tDY0pEmpXluPHj0/buiLKCgiNhMNFaiJBbH19fW7N5Zdf7tZI0smTJ92aSEgo6sPM3OMmcnxGzrvW1tbwuCZz5MgRt2bBggVuzY4dO0Lrmzt3rlsTmS8HBgbcmkhIZmQ+jSwnUhM5fyP7NRISeeDAAbcmEmwZCT5duHChWyPFHn9Z58d07tfJ8AoUAABAJhooAACATDRQAAAAmWigAAAAMtFAAQAAZKKBAgAAyEQDBQAAkIkGCgAAINNZHaQZCUqMhHpFgugefPDB0JjKCoqsNSCs7OWUta7IPouEn0aWEwm9GxoacmuOHTvm1kjSiRMnQnWYGSkl99iKHHuREMCyQmXb2trcmnnz5rk1W7dudWukWJBm5PFHzoXIciL7o6xAzohI2GRkn0XmpkjY5sjIiFsTDVyOHI9lhQVHQkIjj63W/eo+E5vZajP7FzN70sx+YGa/Wty+0My+YWZPF/931zQSACgZ8xeAeom8hXdC0odTSq+U9FpJHzCzSyR9VNI3U0prJX2z+B4AGgnzF4C6cBuolFJPSmlT8fWgpCclrZT0Dkn3FGX3SLqhXoMEgKlg/gJQL1kXkZvZGklXSXpI0rKUUo9UmaQkLS17cABQFuYvAGUKX0FlZp2SviTpQymlQ9GLi81svaT1UxseAMyvbqoAAAseSURBVNSujPkr8scbAM4eoRnBzFpUmXz+JqX098XNvWa2vLh/uaS+8X42pXRHSmldSmldGQMGgBxlzV/T+RepABpf5K/wTNKdkp5MKX2y6q77JN1cfH2zpK+WPzwAmDrmLwD1EnkL7w2S3i3pcTN7rLjtVkkfl/RFM3uPpB2SbqzPEAFgypi/ANSF20CllO6XNNFr128udzjTKxIQFgn+mj9/vlsTDdKMXGcRCZmLjLusILpGE9mvHR0dbk3ksT/xxBNuzSWXXOLWRMfE20h5yp6/vGMiEiYYOa7Kut4qcrxEght7e3tD64sEPEbmr8h2jCxnzpw5pSynrDk3MjdFxnz48GG3JhIWvHfvXrfm/PPPd2uk2OOPiDz+SDhx5LiOHGeT4apIAACATDRQAAAAmWigAAAAMtFAAQAAZKKBAgAAyEQDBQAAkIkGCgAAIBMNFAAAQCYaKAAAgEyRj3I5Y0XSfiNJvpGa/v7+0JgiqeYjIyNuTVmpsJHllJWaXFaKc2Q5w8PDbk1LS4tbs2PHDrdmwYIFbk1UJBEZ9WFmbnp/5NiL1EQSqyPn3dy5c92ayDG1f/9+t0aKpe5HHltZIuuKzKeRVO/Idizr0y/a29tLqTl48KBbMzAw4NZIsQTxyHaMpIyXtc9qxStQAAAAmWigAAAAMtFAAQAAZKKBAgAAyEQDBQAAkIkGCgAAIBMNFAAAQCYaKAAAgExndZBmJGgrElYXCf7yAvhGrVixwq1Zu3atWxMJb4yMKRIOF9mOkbDRSEhmRCSILhL6Njg46NZEgugi+1SKBXcSpDlzUkrusVXW+dLR0eHWROadI0eOlLKcyLkgxebLyPkZORci27G7u7uU8UTWFdnWkXW1tra6NZH5a/Xq1W5NJAj4zW9+s1sTHVPk+Ig8D0RCQiOBtbXOp7wCBQAAkIkGCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEkGYJXnjhBbcmEqAmSVu3bnVrHn/8cbemv7/frYmEmkXCyCKBnGWFbUa248jIiFsTCX3btm2bWxMJz4vUSNJll13m1jzwwAOhZaE+vJC/yHEeOV+OHj3q1kTOha6uLrcmco63tbW5NVLsHI6EF0bmr8i2jmyjyDwYGXOkpqz5NFLT09Pj1kTmykOHDrk1kjRv3jy35vDhw25NWc9LkZpa8QoUAABAJhooAACATDRQAAAAmWigAAAAMtFAAQAAZKKBAgAAyEQDBQAAkIkGCgAAIJObxmVmqyV9TtK5kk5JuiOl9Gkzu03SeyXtLUpvTSn9U70GWg+RkLVI4GIk9C5qYGDArbn33ntLWx/qb/v27aG6c889162ZO3durcM5q5Q9f3nhfGWFO0aCEr1QT0nq6OhwayLBhZF1SdJjjz3m1kTCG1taWtyaoaEht6azs9OtOXbsWCnjOX78uFsT2Y6R0NLIPovU7N+/362JuuKKK9yaTZs2uTVlHY9lBWVPJpJEfkLSh1NKm8ysS9JGM/tGcd+nUkp/XL/hAUBNmL8A1IXbQKWUeiT1FF8PmtmTklbWe2AAUCvmLwD1knUNlJmtkXSVpIeKmz5oZpvN7C4z6y55bABQGuYvAGUKN1Bm1inpS5I+lFI6JOkzki6QdKUqv+F9YoKfW29mj5rZoyWMFwCylTF/TceHkwJoHqEGysxaVJl8/ial9PeSlFLqTSmdTCmdkvRZSVeP97MppTtSSutSSuvKGjQARJU1f0UubgVw9nBnBDMzSXdKejKl9Mmq25dXlf20pCfKHx4ATB3zF4B6ifwV3hskvVvS42Y2+jeqt0q6ycyulJQkbZP0vrqMEACmjvkLQF1E/grvfkk2zl1NlfkE4OzD/AWgXiKvQJ2xFixY4NZ0dXW5NYsWLXJrIsF4UWWFqJ2pF8VGHldZ26e72//jrUjAnhQL/VuyZEloWagP77iJBPxFjqvW1tZSlhOZvyLBntEA1/7+frfm8OHDbk1k3JEAzMj+iJyfkccfWU5kn0X2R6QmElga2RfDw8NujVTePosEU0ceWyRIs/IO/9RxVSQAAEAmGigAAIBMNFAAAACZaKAAAAAy0UABAABkooECAADIRAMFAACQiQYKAAAg01kdpPm9733PrVm8eLFb09vb69b09fWFxhQRCYc7ceJEaes7E0W2YcSBAwfcmg0bNpS2rKeeeiq0LMyMSDBfJOAvEhS4e/fuUta1ceNGt2ZoaMitiYoExkaCbiPLiQROtrS0lLKcsoIbI/u+vb3drYmEds6fP9+tefDBB90aKRbyGwnJjDx3lfXB3rWGSfMKFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMhEAwUAAJCJBgoAACCTlRUoGFqZ2V5J26tuWixp37QNoDzNOG7GPH2acdz1HPN5KSU/Za/BjTN/Sezr6dKMY5aac9yM+XQTzl/T2kC9ZOVmj6aU1s3YAKaoGcfNmKdPM467GcfcCJpxuzHm6dOM42bMcbyFBwAAkIkGCgAAINNMN1B3zPD6p6oZx82Yp08zjrsZx9wImnG7Mebp04zjZsxBM3oNFAAAQDOa6VegAAAAms6MNVBmdr2ZPWVmz5jZR2dqHDnMbJuZPW5mj5nZozM9nomY2V1m1mdmT1TdttDMvmFmTxf/d8/kGMeaYMy3mdkLxfZ+zMzeOpNjHMvMVpvZv5jZk2b2AzP71eL2ht3Wk4y5obd1o2nG+UtqjjmM+Wt6NOP8JTXWHDYjb+GZ2SxJWyVdJ2mXpEck3ZRS+uG0DyaDmW2TtC6l1NAZGWb2I5KGJH0upfSq4rY/ktSfUvp4MeF3p5R+fSbHWW2CMd8maSil9MczObaJmNlySctTSpvMrEvSRkk3SLpFDbqtJxnzz6qBt3Ujadb5S2qOOYz5a3o04/wlNdYcNlOvQF0t6ZmU0nMppRFJfyvpHTM0ljNOSunbkvrH3PwOSfcUX9+jygHXMCYYc0NLKfWklDYVXw9KelLSSjXwtp5kzIhj/qoj5q/p0Yzzl9RYc9hMNVArJe2s+n6XmmMST5K+bmYbzWz9TA8m07KUUo9UOQAlLZ3h8UR90Mw2Fy+RN9RLydXMbI2kqyQ9pCbZ1mPGLDXJtm4AzTp/Sc07hzXFOTWOpjinmnH+kmZ+DpupBsrGua0Z/hzwDSmlV0v6SUkfKF62Rf18RtIFkq6U1CPpEzM7nPGZWaekL0n6UErp0EyPJ2KcMTfFtm4QzTp/Scxh06kpzqlmnL+kxpjDZqqB2iVpddX3qyTtnqGxhKWUdhf/90n6siov5TeL3uK949H3kPtmeDyulFJvSulkSumUpM+qAbe3mbWochL/TUrp74ubG3pbjzfmZtjWDaQp5y+pqeewhj6nxtMM51Qzzl9S48xhM9VAPSJprZm93MxaJf2cpPtmaCwhZtZRXLAmM+uQ9BZJT0z+Uw3lPkk3F1/fLOmrMziWkNGTuPDTarDtbWYm6U5JT6aUPll1V8Nu64nG3OjbusE03fwlNf0c1rDn1EQa/ZxqxvlLaqw5bMaCNIs/Mbxd0ixJd6WUfn9GBhJkZuer8hubJM2W9PlGHbOZ3SvpWlU+obpX0m9L+oqkL0p6maQdkm5MKTXMRY8TjPlaVV6OTZK2SXrf6HvzjcDM3ijp/0l6XNKp4uZbVXk/viG39SRjvkkNvK0bTbPNX1LzzGHMX9OjGecvqbHmMJLIAQAAMpFEDgAAkIkGCgAAIBMNFAAAQCYaKAAAgEw0UAAAAJlooAAAADLRQAEAAGSigQIAAMj0/wHtifv0+qa3bwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "images,_=next(iter(test_loader))\n",
    "images=images.cuda()\n",
    "x_hat, _, _=model_vqvae(images)\n",
    "\n",
    "\n",
    "i=0\n",
    "fig, axes = plt.subplots(1,2, figsize=(10,10))\n",
    "axes[0].imshow(images[i,0].detach().cpu(),cmap='gray')\n",
    "axes[0].set_title('input')\n",
    "\n",
    "axes[1].imshow(x_hat[i,0].detach().cpu(),cmap='gray')\n",
    "axes[1].set_title('Reconstructed from the sampled codes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fe32bf5ee10>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAVcElEQVR4nO3dX4zV5ZkH8O8DDAyMDDgMMwwwQKlgBBOVTIgRXV2bNdYb5aKbctG4iZFe1KRNelFjL+ql2WzbNHHThK6mdNO1MWkNhpjdGtOovVCZUSooIH8EGWaYYfjnDMIIM89ezHEz4vye7+n5nXN+Z32/n4TMcB7ec97zO+fh/Hl+z/uau0NEvv5mFT0BEakPJbtIIpTsIolQsoskQskukog59byx2bNn+5w5tblJMwvjrOrAxrN4pKmpqeKxALBw4cIwfvXq1czYxYsXw7F57lc1xufR3NwcxltbWyu+7rzPp88++yyMj42NVXzdUfzatWuYmJiYcfK5Ms/MHgTwKwCzAfyHuz8T/fs5c+Zg+fLlFd/erFnZb0SiGBAnBDA1t0iehF2xYkUYZ0+se+65J4yfOXMmM7Z79+5wbN7/fNn4KD45ORmOZY/pTTfdFMYfeOCBzNjs2bPDsezxvnLlShjv6+sL42+//XbF1x0dt/7+/sxYxW/jzWw2gH8H8G0AGwBsM7MNlV6fiNRWns/smwEccfdj7v45gD8AeLg60xKRasuT7CsAnJz29/7SZV9iZtvNrNfMeicmJnLcnIjkkSfZZ/qg+ZVvDtx9h7v3uHsP+5wkIrWTJ9n7AXRP+/tKAAP5piMitZIn2fcAWGdm3zCzuQC+C+Dl6kxLRKqt4rqLu18zsycA/A+mSm/Pu/sHZYzLngwp47D6YmR8fLzi6waA6PsGVtabO3duGN+6dWsYf/zxx8P4iy++mBl79tlnw7Fr164N46w8xo5b9NGNXffKlSvD+F133RXGb7311szY8PBwOHb+/PlhvL29PYwfO3YsjJ8/fz4zxo4Lez5lyVVkdfdXALyS5zpEpD50uqxIIpTsIolQsoskQskukgglu0gilOwiiah3PzsWLVqUGWf1xagO//nnn4djWbsk642O+pNZO+SJEyfC+NDQUBg/ffp0GP/kk08yY+zcBVazZe2WrD133rx5mTH2mLA+fnbuxN69ezNjHR0d4Vh2XEZHR8P4qVOnwng092XLloVjo174iF7ZRRKhZBdJhJJdJBFKdpFEKNlFEqFkF0lEXUtvk5OTYQmLrRa6evXqzBhbMpm1HJ47dy6MRyUsVqZZsGBBGGclqLfeeiuMR6W3G264IRzLSo4tLS1hnB33aDVhVrJct25dGGei62ePGVtVaWRkJIyzFYWj1mDWNsxK1Fn0yi6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIomoa53d3cMln1md/d57782M7d+/PxwbLd0L8C12o3mzmm3U1gvwlsWjR49WPJ7NjR0XVqdnNd+ozn7LLbeEY9nc2bkRUS2cPSaDg4Nh/NNPPw3jN954Yxjv7u7OjLHnIjsvI3NcRaNE5P8dJbtIIpTsIolQsoskQskukgglu0gilOwiiahrnd3Mwr7wS5cuheOjPl+27DCre0ZbMgPxUtWs97m1tTWMs5ot62+Olppmt83qzawOz3r1u7q6MmPRMtMAcPbs2TDOlrmOzo1gS2CzY86w475x48bM2MGDB8Ox0RoC0f3KlexmdhzAKIAJANfcvSfP9YlI7VTjlf0f3T1etkNECqfP7CKJyJvsDuDPZtZnZttn+gdmtt3Mes2sl30uFpHayfs2fou7D5hZB4BXzeygu78x/R+4+w4AOwCgubk537ceIlKxXK/s7j5Q+jkM4CUAm6sxKRGpvoqT3cxazGzhF78DeABA3GcqIoXJ8za+E8BLpbreHAD/5e7/HQ1w97D/mfXp5vnMH9V7AaC9vT2MRzVdNi/WnzwwMBDG29rawnhUr2b96Eze8xeGh4czY6xnnJ0DcObMmTAePdduu+22cCx7TC9fvhzG2XG5++67w3ikr68vM1aTOru7HwMQHzERaRgqvYkkQskukgglu0gilOwiiVCyiySiri2us2bNClsi2Ra+UcsiaykcHx8P42x81H7LSkDsulmZhx2X+fPnVxQDgJMnT4ZxVg5l5bGodMdamtkS2xcuXKjZbbP22ajlGeDl1sWLF2fGouW3AeDw4cOZsejx0iu7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskou519ubm5sw4q4VHSy6zWjWrN0dLXANxiyyrNbNli6OlgQE+t2i5Z1arZi2w0eMF8FbOjo6OzBir4UfnVQC81h1hdXS2hDaLX716NYxH21Gz+x3V8MMW8vBaReRrQ8kukgglu0gilOwiiVCyiyRCyS6SCCW7SCLqWmefnJwM65us9hn1XrOlolk9mImWc2b95i0tLWGc1bJZHT5a1pjVe9k5AOz8BXYOQISdV8GOG1vmOpo7q2WzpcXZbXd2dobxaJtvVsOPzo2Irlev7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukoi61tmbmpqwdOnSzHjU4wsAo6OjmbFoHW6Ar8XNat1RXZbV8Fmtm23/G/XxA/m2so7qsgDCdf6BeLtoIF6fnZ1Xwc4BiHq3gXhdefZcY732bH0Edv3ROgNszXk2t8xx7B+Y2fNmNmxm+6dd1mZmr5rZ4dLPfGesiEjNlfNfxG8BPHjdZU8CeM3d1wF4rfR3EWlgNNnd/Q0A5667+GEAO0u/7wTwSJXnJSJVVuln9k53HwQAdx80s8yFxsxsO4DtAP98JyK1U/Nv4919h7v3uHsP+9JCRGqn0mQfMrMuACj9HK7elESkFipN9pcBPFr6/VEAu6ozHRGpFfqZ3cxeAHAfgHYz6wfwMwDPAHjRzB4D8AmA75RzY+4e1pxZPXl4OPsNBBvL+t1ZT3pU82UfT1gdPU9PODB1XLOwOjr7HoUdV7Zmfn9/f8W3zUTPBwAYHBzMjJ07d/13zl/Gavh51zCIzgGI1m0A4rlF51zQZ5m7b8sIfYuNFZHGodNlRRKhZBdJhJJdJBFKdpFEKNlFElHXFlczC0sWUTskwMtAEVb+YiWkaG6sxZUtDfzRRx+F8dOnT4fxkZGRzFhUlgN4aY4tucyOa2tra2aMzY217rK25Oj6T5w4keu6WbmVPWaRPFtVa8tmEVGyi6RCyS6SCCW7SCKU7CKJULKLJELJLpKIutbZJyYmwuWgo22RgbglktXoWT052gYXiOumBw4cCMceO3YsjL/++uthPGrVBOJ6NLtfrKbL6s3sHIONGzdmxvJsuQzwNtPjx4+H8UhUywZ4nZ2dExItF82OS3Tb0TLTemUXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFE1L2fPaqVs7pptAw1q6OzbXDZ0sKHDh3KjOXZMhngNd2OjszdtXLffnTeA8Dr9Ex036IafDnx3bt3h/Ho3AjWh8967fPW4dnzNRL1rEfz1iu7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskoq519snJybDvPNoWmVmyZEkYj/p8AV5PjvqTx8fHw7GbNm0K42wN86GhoTB+8eLFzFi0bjvAa/Tz588P42xuUa9+dO4CAHR2dobx9957L4xHvfirVq0Kx7Ln4tmzZ8M4Oy5jY2OZMXa+SbQGQa46u5k9b2bDZrZ/2mVPm9kpM9tb+vMQux4RKVY5b+N/C+DBGS7/pbvfXvrzSnWnJSLVRpPd3d8AEJ9LKiINL88XdE+Y2fult/mZC5GZ2XYz6zWzXrbemYjUTqXJ/msA3wRwO4BBAD/P+ofuvsPde9y9Z86cun4fKCLTVJTs7j7k7hPuPgngNwA2V3daIlJtFSW7mXVN++tWAPuz/q2INAb6vtrMXgBwH4B2M+sH8DMA95nZ7QAcwHEA3y/nxiYnJ8P6YtTrDsT1bLYf9ptvvhnGd+3aFcY//PDDzNjSpUvDsevWrQvjy5cvD+OnTp0K44sXL86MsfXL29vbwzj76BXV+IH4cent7Q3HLliwIIyzXvxoHQB2XgXrpY/WVgCAjz/+OIxHx5Ud80q/+6LJ7u7bZrj4uYpuTUQKo9NlRRKhZBdJhJJdJBFKdpFEKNlFElHXU9rcPVwGl5UUWJtqZGBgIIyPjIyE8WhubEtl1oq5ZcuWMH7w4MEwHrWpsvIVWxKZbcnMSm9Ri2yeVk4AWLZsWRiP7htbKpqVQ4eHh8M420J80aJFmbFanWmqV3aRRCjZRRKhZBdJhJJdJBFKdpFEKNlFEqFkF0lEXevss2bNCuu+rGUxqsuy2iTbkpltobtw4cLMGNsOmtWi2ZLJ7L5Fbayslh3dr3LGs62JK132GODnALA21ei8DLZEdldXVxhn52Ww1uGoDs/ud7REdnSf9coukgglu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJqHs/+5UrVzLj0ZLIX4zPwnrd2TLVbOviaAtfVqtmPeU333xzGGf3LaoZd3d3h2OZtra2MM7q7H19fZkxdm4Dm/vq1avDePSYsy2Z2f1i/e7r168P49Fjys4JiZax7u/vz77N8FpF5GtDyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIupaZwfi2ipbJzyqR7N1vKP16ssZH/WUs3mz7X1Z/3KeXv2VK1eGY1taWsI4q0ezvu9NmzZVfN1r1qwJ42zu0bbJbK1/tk02e0zY3KM8YDX8I0eOZMZy9bObWbeZ/cXMDpjZB2b2w9LlbWb2qpkdLv2MdxMQkUKV8zb+GoAfu/stAO4E8AMz2wDgSQCvufs6AK+V/i4iDYomu7sPuvu7pd9HARwAsALAwwB2lv7ZTgCP1GqSIpLf3/WZ3czWALgDwNsAOt19EJj6D8HMOjLGbAewHajdHlYiwpX9bbyZ3QDgjwB+5O7ZKxxex913uHuPu/ewxgcRqZ2ykt3MmjCV6L939z+VLh4ys65SvAtA/HW2iBSKvq+2qfrIcwAOuPsvpoVeBvAogGdKP3eVcV1huWXz5s3h+Pvvvz8zFrVSAsChQ4fCeLQcMxC337KPJ3feeWcYj5YGLicelVvYuynWVsxaPdnc7rjjjswYO26stHb48OEwfubMmcwYK/tFYwG+XXSelmvWMr1nz57MWNSqXc6H6C0Avgdgn5ntLV32FKaS/EUzewzAJwC+U8Z1iUhBaLK7+18BZP03+K3qTkdEakWny4okQskukgglu0gilOwiiVCyiySi7uevRrXVqB0SiOvsrJZ98uTJMM5qtq2trZkx1g7JarLnz58P46wFNlqqOpo3wOvsDKuzR3X+y5cvh2MvXLgQxlktfMmSJZkxts02m1u0ZDMAjI2NhfHocWHLmkdzj1q59coukgglu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJqGudfe7cuVixYkVmnNWToyV0WW80q6t2dnaG8Wirada7fOnSpTAebUUN8Jpt1HPOlnpetGhRGGc95awWHp1DwB4T9piy+xZtZc3q6KzfnT1mrFZ++vTpMB5pamrKjEXz1iu7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskoqH2Y9q3b1/FY9nWxKxWzbZ0Hh0dzYwNDQ2FY1etWhXGWc2X9aSzNe8jrB7MsHpzVIcfGRkJx0b96AA/NyLq1WfXze4XO3eCPSbRcWfPxWiL8GjeemUXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFElLM/ezeA3wFYBmASwA53/5WZPQ3gcQBfLN79lLu/El3X+Pg4jh49mhln66+Pj49nxlhvc9QDDPB9yDs6OjJjAwMD4VjWE87iGzZsCOPRcWN7fbP7HR3zckT3jfXCs+PC5h495qyWzc4/iNbqLycerYHAbrvSfvZyTqq5BuDH7v6umS0E0Gdmr5Ziv3T3fyvjOkSkYOXszz4IYLD0+6iZHQCQvdyMiDSkv+szu5mtAXAHgLdLFz1hZu+b2fNmdmPGmO1m1mtmvXlPzRSRypWd7GZ2A4A/AviRu38K4NcAvgngdky98v98pnHuvsPde9y9J9r3S0Rqq6xkN7MmTCX67939TwDg7kPuPuHukwB+A2Bz7aYpInnRZLepr/eeA3DA3X8x7fLpX39vBbC/+tMTkWop59v4LQC+B2Cfme0tXfYUgG1mdjsAB3AcwPfZFZkZ5s2blxm/ePFiOP6dd97JjLGWQ9Ymunbt2jAetcj29vaGY9l20evXrw/jbNnhaAnuqB0SAJYvXx7GWXnr6tWrYTxqM2Xl0Pb29jDe1tZW8W2zlme2FTW739Ey1kDcihq1UwPxVtW5Sm/u/lcAM11DWFMXkcaiM+hEEqFkF0mEkl0kEUp2kUQo2UUSoWQXSYSxJXOrqbm52bu7uzPjrCYc1ejZWFbTja4biFs9WWsua79lbaisphvVwlm9mC3HHLX2AnzL56juy1pcWZ2dbekcxVmdPe822mz78Tytw4cOHcqM7du3D2NjYzMedL2yiyRCyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIupaZzezMwBOTLuoHUC8b29xGnVujTovQHOrVDXnttrdl84UqGuyf+XGzXrdvaewCQQadW6NOi9Ac6tUveamt/EiiVCyiySi6GTfUfDtRxp1bo06L0Bzq1Rd5lboZ3YRqZ+iX9lFpE6U7CKJKCTZzexBMztkZkfM7Mki5pDFzI6b2T4z22tm8YLwtZ/L82Y2bGb7p13WZmavmtnh0s8Z99graG5Pm9mp0rHba2YPFTS3bjP7i5kdMLMPzOyHpcsLPXbBvOpy3Or+md3MZgP4CMA/AegHsAfANnf/sK4TyWBmxwH0uHvhJ2CY2T8AGAPwO3e/tXTZvwI45+7PlP6jvNHdf9Igc3sawFjR23iXdivqmr7NOIBHAPwLCjx2wbz+GXU4bkW8sm8GcMTdj7n75wD+AODhAubR8Nz9DQDnrrv4YQA7S7/vxNSTpe4y5tYQ3H3Q3d8t/T4K4Ittxgs9dsG86qKIZF8BYPp+SP1orP3eHcCfzazPzLYXPZkZdLr7IDD15AEQrxtVf3Qb73q6bpvxhjl2lWx/nlcRyT7T+liNVP/b4u6bAHwbwA9Kb1elPGVt410vM2wz3hAq3f48ryKSvR/A9FUnVwIYKGAeM3L3gdLPYQAvofG2oh76Ygfd0s/hgufzfxppG++ZthlHAxy7Irc/LyLZ9wBYZ2bfMLO5AL4L4OUC5vEVZtZS+uIEZtYC4AE03lbULwN4tPT7owB2FTiXL2mUbbyzthlHwceu8O3P3b3ufwA8hKlv5I8C+GkRc8iY11oAfyv9+aDouQF4AVNv665i6h3RYwCWAHgNwOHSz7YGmtt/AtgH4H1MJVZXQXO7G1MfDd8HsLf056Gij10wr7ocN50uK5IInUEnkgglu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJ+F87SO9zurV2yAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "randind=torch.randint_like(indices,high=num_embeddings)\n",
    "\n",
    "out=model_vqvae.compute_out_from_indices(randind)\n",
    "\n",
    "plt.imshow(out[0,0].detach().cpu(),cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'Gated-PixelCNN-Pytorch'...\n",
      "remote: Enumerating objects: 49, done.\u001b[K\n",
      "remote: Counting objects: 100% (49/49), done.\u001b[K\n",
      "remote: Compressing objects: 100% (44/44), done.\u001b[K\n",
      "remote: Total 49 (delta 26), reused 12 (delta 5), pack-reused 0\u001b[K\n",
      "Unpacking objects: 100% (49/49), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/rogertrullo/Gated-PixelCNN-Pytorch.git\n",
    "!cp Gated-PixelCNN-Pytorch/*py ./\n",
    "!rm -r Gated-PixelCNN-Pytorch/\n",
    "\n",
    "from GatedPixelCNN import PixelCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlayers=10 # number of layers for pixelcnn\n",
    "inchans=1 #number of input channels (currently only one is supported)\n",
    "nfeats=256 #number of feature maps across the network\n",
    "Klevels=num_embeddings #number of levels to use in discretization\n",
    "nepochs=5000 #number of epochs to train\n",
    "lr=1e-3 #learning rate for optimizer\n",
    "generate_every=300\n",
    "nimgs_to_generate=16\n",
    "\n",
    "def generate_imgs(model, shape, nimgs):\n",
    "    x=torch.zeros((nimgs,1,shape[0],shape[1])).cuda()\n",
    "    logits=model(x)\n",
    "    model.eval()\n",
    "    for i in range(x.shape[2]):\n",
    "        for j in range(x.shape[3]):\n",
    "            logits=model(x)\n",
    "            probs=torch.softmax(logits[:,:,i,j],1)\n",
    "            sample=probs.multinomial(1)\n",
    "            x[:,:,i,j]=sample.float()/(Klevels-1)\n",
    "    model.train()\n",
    "    return x.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_prior=PixelCNN(nlayers=nlayers, in_channels=inchans, nfeats=nfeats, Klevels=Klevels, ksz_A=5).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, it:0/938, loss:4.856324672698975\n",
      "epoch: 0, it:100/938, loss:4.097647190093994\n",
      "epoch: 0, it:200/938, loss:3.9334638118743896\n",
      "epoch: 0, it:300/938, loss:3.7523956298828125\n",
      "epoch: 0, it:400/938, loss:3.710120439529419\n",
      "epoch: 0, it:500/938, loss:3.679757833480835\n",
      "epoch: 0, it:600/938, loss:3.6030361652374268\n",
      "epoch: 0, it:700/938, loss:3.532132625579834\n",
      "epoch: 0, it:800/938, loss:3.493411064147949\n",
      "epoch: 0, it:900/938, loss:3.2697126865386963\n",
      "epoch: 1, it:0/938, loss:3.333106279373169\n",
      "epoch: 1, it:100/938, loss:3.272444725036621\n",
      "epoch: 1, it:200/938, loss:3.156203269958496\n",
      "epoch: 1, it:300/938, loss:3.364394187927246\n",
      "epoch: 1, it:400/938, loss:3.1038002967834473\n",
      "epoch: 1, it:500/938, loss:3.1167447566986084\n",
      "epoch: 1, it:600/938, loss:3.202674627304077\n",
      "epoch: 1, it:700/938, loss:3.0217032432556152\n",
      "epoch: 1, it:800/938, loss:2.93473219871521\n",
      "epoch: 1, it:900/938, loss:3.1099650859832764\n",
      "epoch: 2, it:0/938, loss:3.1799542903900146\n",
      "epoch: 2, it:100/938, loss:2.9020473957061768\n",
      "epoch: 2, it:200/938, loss:2.9498109817504883\n",
      "epoch: 2, it:300/938, loss:2.7442502975463867\n",
      "epoch: 2, it:400/938, loss:3.1953701972961426\n",
      "epoch: 2, it:500/938, loss:3.074913501739502\n",
      "epoch: 2, it:600/938, loss:3.063185453414917\n",
      "epoch: 2, it:700/938, loss:2.8038339614868164\n",
      "epoch: 2, it:800/938, loss:2.8565452098846436\n",
      "epoch: 2, it:900/938, loss:2.7904512882232666\n",
      "epoch: 3, it:0/938, loss:3.0099916458129883\n",
      "epoch: 3, it:100/938, loss:2.8848633766174316\n",
      "epoch: 3, it:200/938, loss:2.9922444820404053\n",
      "epoch: 3, it:300/938, loss:2.81662917137146\n",
      "epoch: 3, it:400/938, loss:2.8134403228759766\n",
      "epoch: 3, it:500/938, loss:2.6527345180511475\n",
      "epoch: 3, it:600/938, loss:2.9594006538391113\n",
      "epoch: 3, it:700/938, loss:2.9783966541290283\n",
      "epoch: 3, it:800/938, loss:2.6962525844573975\n",
      "epoch: 3, it:900/938, loss:2.8249759674072266\n",
      "epoch: 4, it:0/938, loss:2.988297700881958\n",
      "epoch: 4, it:100/938, loss:3.045316457748413\n",
      "epoch: 4, it:200/938, loss:2.744182586669922\n",
      "epoch: 4, it:300/938, loss:2.668686628341675\n",
      "epoch: 4, it:400/938, loss:2.8138022422790527\n",
      "epoch: 4, it:500/938, loss:2.9282548427581787\n",
      "epoch: 4, it:600/938, loss:2.8604865074157715\n",
      "epoch: 4, it:700/938, loss:2.7222115993499756\n",
      "epoch: 4, it:800/938, loss:2.8460004329681396\n",
      "epoch: 4, it:900/938, loss:2.68696928024292\n",
      "epoch: 5, it:0/938, loss:2.7253191471099854\n",
      "epoch: 5, it:100/938, loss:3.1163153648376465\n",
      "epoch: 5, it:200/938, loss:2.6114561557769775\n",
      "epoch: 5, it:300/938, loss:2.6996448040008545\n",
      "epoch: 5, it:400/938, loss:2.674077033996582\n",
      "epoch: 5, it:500/938, loss:2.973508834838867\n",
      "epoch: 5, it:600/938, loss:2.7450742721557617\n",
      "epoch: 5, it:700/938, loss:2.764235496520996\n",
      "epoch: 5, it:800/938, loss:2.850834846496582\n",
      "epoch: 5, it:900/938, loss:2.8139548301696777\n",
      "epoch: 6, it:0/938, loss:2.7482638359069824\n",
      "epoch: 6, it:100/938, loss:2.7607123851776123\n",
      "epoch: 6, it:200/938, loss:2.8087637424468994\n",
      "epoch: 6, it:300/938, loss:2.684077262878418\n",
      "epoch: 6, it:400/938, loss:2.5998010635375977\n",
      "epoch: 6, it:500/938, loss:2.491323471069336\n",
      "epoch: 6, it:600/938, loss:2.8329384326934814\n",
      "epoch: 6, it:700/938, loss:2.7483246326446533\n",
      "epoch: 6, it:800/938, loss:2.730050563812256\n",
      "epoch: 6, it:900/938, loss:2.7936289310455322\n",
      "epoch: 7, it:0/938, loss:2.6844000816345215\n",
      "epoch: 7, it:100/938, loss:2.800490140914917\n",
      "epoch: 7, it:200/938, loss:2.795322895050049\n",
      "epoch: 7, it:300/938, loss:2.658883810043335\n",
      "epoch: 7, it:400/938, loss:2.719313621520996\n",
      "epoch: 7, it:500/938, loss:2.6165971755981445\n",
      "epoch: 7, it:600/938, loss:2.530362129211426\n",
      "epoch: 7, it:700/938, loss:2.7881624698638916\n",
      "epoch: 7, it:800/938, loss:2.8273308277130127\n",
      "epoch: 7, it:900/938, loss:2.6378719806671143\n",
      "epoch: 8, it:0/938, loss:2.7493793964385986\n",
      "epoch: 8, it:100/938, loss:2.73956298828125\n",
      "epoch: 8, it:200/938, loss:2.2938196659088135\n",
      "epoch: 8, it:300/938, loss:2.6878719329833984\n",
      "epoch: 8, it:400/938, loss:2.6740944385528564\n",
      "epoch: 8, it:500/938, loss:2.7548983097076416\n",
      "epoch: 8, it:600/938, loss:2.783809185028076\n",
      "epoch: 8, it:700/938, loss:2.611034631729126\n",
      "epoch: 8, it:800/938, loss:2.604112386703491\n",
      "epoch: 8, it:900/938, loss:2.6654181480407715\n",
      "epoch: 9, it:0/938, loss:2.6885905265808105\n",
      "epoch: 9, it:100/938, loss:2.4058876037597656\n",
      "epoch: 9, it:200/938, loss:2.689255714416504\n",
      "epoch: 9, it:300/938, loss:2.5699119567871094\n",
      "epoch: 9, it:400/938, loss:2.7263550758361816\n",
      "epoch: 9, it:500/938, loss:2.481454849243164\n",
      "epoch: 9, it:600/938, loss:2.739328384399414\n",
      "epoch: 9, it:700/938, loss:2.628577709197998\n",
      "epoch: 9, it:800/938, loss:2.648404121398926\n",
      "epoch: 9, it:900/938, loss:2.6073076725006104\n",
      "epoch: 10, it:0/938, loss:2.564117431640625\n",
      "epoch: 10, it:100/938, loss:2.577298641204834\n",
      "epoch: 10, it:200/938, loss:2.7224018573760986\n",
      "epoch: 10, it:300/938, loss:2.513246536254883\n",
      "epoch: 10, it:400/938, loss:2.6865828037261963\n",
      "epoch: 10, it:500/938, loss:2.6332614421844482\n",
      "epoch: 10, it:600/938, loss:2.572540044784546\n",
      "epoch: 10, it:700/938, loss:2.6419179439544678\n",
      "epoch: 10, it:800/938, loss:2.7023234367370605\n",
      "epoch: 10, it:900/938, loss:2.551818609237671\n",
      "epoch: 11, it:0/938, loss:2.7019248008728027\n",
      "epoch: 11, it:100/938, loss:2.5571300983428955\n",
      "epoch: 11, it:200/938, loss:2.60573673248291\n",
      "epoch: 11, it:300/938, loss:2.5942928791046143\n",
      "epoch: 11, it:400/938, loss:2.6693341732025146\n",
      "epoch: 11, it:500/938, loss:2.697319746017456\n",
      "epoch: 11, it:600/938, loss:2.5504841804504395\n",
      "epoch: 11, it:700/938, loss:2.561589241027832\n",
      "epoch: 11, it:800/938, loss:2.6960489749908447\n",
      "epoch: 11, it:900/938, loss:2.552037239074707\n",
      "epoch: 12, it:0/938, loss:2.5736870765686035\n",
      "epoch: 12, it:100/938, loss:2.572338104248047\n",
      "epoch: 12, it:200/938, loss:2.6463310718536377\n",
      "epoch: 12, it:300/938, loss:2.807178020477295\n",
      "epoch: 12, it:400/938, loss:2.5915842056274414\n",
      "epoch: 12, it:500/938, loss:2.554337739944458\n",
      "epoch: 12, it:600/938, loss:2.607942581176758\n",
      "epoch: 12, it:700/938, loss:2.4498956203460693\n",
      "epoch: 12, it:800/938, loss:2.434438705444336\n",
      "epoch: 12, it:900/938, loss:2.6182258129119873\n",
      "epoch: 13, it:0/938, loss:2.5239346027374268\n",
      "epoch: 13, it:100/938, loss:2.6388254165649414\n",
      "epoch: 13, it:200/938, loss:2.6390464305877686\n",
      "epoch: 13, it:300/938, loss:2.5510897636413574\n",
      "epoch: 13, it:400/938, loss:2.655101776123047\n",
      "epoch: 13, it:500/938, loss:2.4030439853668213\n",
      "epoch: 13, it:600/938, loss:2.6295948028564453\n",
      "epoch: 13, it:700/938, loss:2.579955577850342\n",
      "epoch: 13, it:800/938, loss:2.6765503883361816\n",
      "epoch: 13, it:900/938, loss:2.708150863647461\n",
      "epoch: 14, it:0/938, loss:2.4768292903900146\n",
      "epoch: 14, it:100/938, loss:2.6405961513519287\n",
      "epoch: 14, it:200/938, loss:2.713233709335327\n",
      "epoch: 14, it:300/938, loss:2.5117390155792236\n",
      "epoch: 14, it:400/938, loss:2.4426584243774414\n",
      "epoch: 14, it:500/938, loss:2.6327707767486572\n",
      "epoch: 14, it:600/938, loss:2.5304439067840576\n",
      "epoch: 14, it:700/938, loss:2.518404483795166\n",
      "epoch: 14, it:800/938, loss:2.728813886642456\n",
      "epoch: 14, it:900/938, loss:2.4948415756225586\n",
      "epoch: 15, it:0/938, loss:2.5347375869750977\n",
      "epoch: 15, it:100/938, loss:2.547163724899292\n",
      "epoch: 15, it:200/938, loss:2.4557018280029297\n",
      "epoch: 15, it:300/938, loss:2.578873872756958\n",
      "epoch: 15, it:400/938, loss:2.5679306983947754\n",
      "epoch: 15, it:500/938, loss:2.482560873031616\n",
      "epoch: 15, it:600/938, loss:2.535198450088501\n",
      "epoch: 15, it:700/938, loss:2.540499687194824\n",
      "epoch: 15, it:800/938, loss:2.4348862171173096\n",
      "epoch: 15, it:900/938, loss:2.545863628387451\n",
      "Epoch    16: reducing learning rate of group 0 to 2.0000e-04.\n",
      "epoch: 16, it:0/938, loss:2.5430185794830322\n",
      "epoch: 16, it:100/938, loss:2.3872199058532715\n",
      "epoch: 16, it:200/938, loss:2.392273187637329\n",
      "epoch: 16, it:300/938, loss:2.4310853481292725\n",
      "epoch: 16, it:400/938, loss:2.4516098499298096\n",
      "epoch: 16, it:500/938, loss:2.410102605819702\n",
      "epoch: 16, it:600/938, loss:2.505340576171875\n",
      "epoch: 16, it:700/938, loss:2.465730905532837\n",
      "epoch: 16, it:800/938, loss:2.307797431945801\n",
      "epoch: 16, it:900/938, loss:2.398446798324585\n",
      "epoch: 17, it:0/938, loss:2.3575448989868164\n",
      "epoch: 17, it:100/938, loss:2.3990423679351807\n",
      "epoch: 17, it:200/938, loss:2.447559356689453\n",
      "epoch: 17, it:300/938, loss:2.2124991416931152\n",
      "epoch: 17, it:400/938, loss:2.26011061668396\n",
      "epoch: 17, it:500/938, loss:2.3228635787963867\n",
      "epoch: 17, it:600/938, loss:2.2788989543914795\n",
      "epoch: 17, it:700/938, loss:2.430457592010498\n",
      "epoch: 17, it:800/938, loss:2.3948609828948975\n",
      "epoch: 17, it:900/938, loss:2.3350651264190674\n",
      "epoch: 18, it:0/938, loss:2.3016598224639893\n",
      "epoch: 18, it:100/938, loss:2.2354519367218018\n",
      "epoch: 18, it:200/938, loss:2.2955920696258545\n",
      "epoch: 18, it:300/938, loss:2.384145975112915\n",
      "epoch: 18, it:400/938, loss:2.2845664024353027\n",
      "epoch: 18, it:500/938, loss:2.121623992919922\n",
      "epoch: 18, it:600/938, loss:2.3835248947143555\n",
      "epoch: 18, it:700/938, loss:2.2662267684936523\n",
      "epoch: 18, it:800/938, loss:2.3535332679748535\n",
      "epoch: 18, it:900/938, loss:2.3541314601898193\n",
      "epoch: 19, it:0/938, loss:2.2339775562286377\n",
      "epoch: 19, it:100/938, loss:2.273556709289551\n",
      "epoch: 19, it:200/938, loss:2.355372905731201\n",
      "epoch: 19, it:300/938, loss:2.1057000160217285\n",
      "epoch: 19, it:400/938, loss:2.181384563446045\n",
      "epoch: 19, it:500/938, loss:2.224303722381592\n",
      "epoch: 19, it:600/938, loss:2.175668478012085\n",
      "epoch: 19, it:700/938, loss:2.251203775405884\n",
      "epoch: 19, it:800/938, loss:2.3002474308013916\n",
      "epoch: 19, it:900/938, loss:2.1982288360595703\n",
      "epoch: 20, it:0/938, loss:2.258187770843506\n",
      "epoch: 20, it:100/938, loss:2.2006139755249023\n",
      "epoch: 20, it:200/938, loss:2.1440799236297607\n",
      "epoch: 20, it:300/938, loss:2.1755151748657227\n",
      "epoch: 20, it:400/938, loss:2.217921018600464\n",
      "epoch: 20, it:500/938, loss:2.1549832820892334\n",
      "epoch: 20, it:600/938, loss:2.270822763442993\n",
      "epoch: 20, it:700/938, loss:2.2232089042663574\n",
      "epoch: 20, it:800/938, loss:2.3004660606384277\n",
      "epoch: 20, it:900/938, loss:2.2470521926879883\n",
      "epoch: 21, it:0/938, loss:2.1133201122283936\n",
      "epoch: 21, it:100/938, loss:2.1688637733459473\n",
      "epoch: 21, it:200/938, loss:2.2620036602020264\n",
      "epoch: 21, it:300/938, loss:2.334702730178833\n",
      "epoch: 21, it:400/938, loss:2.1043713092803955\n",
      "epoch: 21, it:500/938, loss:2.104484796524048\n",
      "epoch: 21, it:600/938, loss:2.0993030071258545\n",
      "epoch: 21, it:700/938, loss:2.1662955284118652\n",
      "epoch: 21, it:800/938, loss:2.2861502170562744\n",
      "epoch: 21, it:900/938, loss:2.1492533683776855\n",
      "epoch: 22, it:0/938, loss:2.2085208892822266\n",
      "epoch: 22, it:100/938, loss:2.104948043823242\n",
      "epoch: 22, it:200/938, loss:2.127636671066284\n",
      "epoch: 22, it:300/938, loss:2.245657205581665\n",
      "epoch: 22, it:400/938, loss:2.2074594497680664\n",
      "epoch: 22, it:500/938, loss:2.027353525161743\n",
      "epoch: 22, it:600/938, loss:2.1118369102478027\n",
      "epoch: 22, it:700/938, loss:2.095259428024292\n",
      "epoch: 22, it:800/938, loss:2.1204423904418945\n",
      "epoch: 22, it:900/938, loss:2.027700424194336\n",
      "epoch: 23, it:0/938, loss:2.1645100116729736\n",
      "epoch: 23, it:100/938, loss:2.234598159790039\n",
      "epoch: 23, it:200/938, loss:2.1812448501586914\n",
      "epoch: 23, it:300/938, loss:2.132321834564209\n",
      "epoch: 23, it:400/938, loss:2.0507290363311768\n",
      "epoch: 23, it:500/938, loss:1.9278422594070435\n",
      "epoch: 23, it:600/938, loss:2.20373272895813\n",
      "epoch: 23, it:700/938, loss:2.2495949268341064\n",
      "epoch: 23, it:800/938, loss:2.242295026779175\n",
      "epoch: 23, it:900/938, loss:2.178013324737549\n",
      "epoch: 24, it:0/938, loss:2.0765180587768555\n",
      "epoch: 24, it:100/938, loss:1.979123830795288\n",
      "epoch: 24, it:200/938, loss:2.0057976245880127\n",
      "epoch: 24, it:300/938, loss:2.055872678756714\n",
      "epoch: 24, it:400/938, loss:2.0766868591308594\n",
      "epoch: 24, it:500/938, loss:2.1068239212036133\n",
      "epoch: 24, it:600/938, loss:2.070044994354248\n",
      "epoch: 24, it:700/938, loss:2.0941367149353027\n",
      "epoch: 24, it:800/938, loss:2.073093891143799\n",
      "epoch: 24, it:900/938, loss:2.032517194747925\n",
      "epoch: 25, it:0/938, loss:2.053826093673706\n",
      "epoch: 25, it:100/938, loss:2.0327095985412598\n",
      "epoch: 25, it:200/938, loss:2.040950059890747\n",
      "epoch: 25, it:300/938, loss:2.072514533996582\n",
      "epoch: 25, it:400/938, loss:2.0001237392425537\n",
      "epoch: 25, it:500/938, loss:2.0394816398620605\n",
      "epoch: 25, it:600/938, loss:1.9968538284301758\n",
      "epoch: 25, it:700/938, loss:2.170661687850952\n",
      "epoch: 25, it:800/938, loss:1.9107985496520996\n",
      "epoch: 25, it:900/938, loss:2.041027784347534\n",
      "epoch: 26, it:0/938, loss:2.099195718765259\n",
      "epoch: 26, it:100/938, loss:2.149937152862549\n",
      "epoch: 26, it:200/938, loss:2.1365394592285156\n",
      "epoch: 26, it:300/938, loss:1.9705396890640259\n",
      "epoch: 26, it:400/938, loss:2.0743703842163086\n",
      "epoch: 26, it:500/938, loss:2.01558518409729\n",
      "epoch: 26, it:600/938, loss:2.0474939346313477\n",
      "epoch: 26, it:700/938, loss:1.9955084323883057\n",
      "epoch: 26, it:800/938, loss:1.9531712532043457\n",
      "epoch: 26, it:900/938, loss:2.028654098510742\n",
      "epoch: 27, it:0/938, loss:2.0575857162475586\n",
      "epoch: 27, it:100/938, loss:1.8915573358535767\n",
      "epoch: 27, it:200/938, loss:1.878806233406067\n",
      "epoch: 27, it:300/938, loss:1.9979867935180664\n",
      "epoch: 27, it:400/938, loss:1.9184625148773193\n",
      "epoch: 27, it:500/938, loss:2.128725528717041\n",
      "epoch: 27, it:600/938, loss:2.010206699371338\n",
      "epoch: 27, it:700/938, loss:1.8821945190429688\n",
      "epoch: 27, it:800/938, loss:2.0350637435913086\n",
      "epoch: 27, it:900/938, loss:2.0795347690582275\n",
      "epoch: 28, it:0/938, loss:1.8400803804397583\n",
      "epoch: 28, it:100/938, loss:1.8951069116592407\n",
      "epoch: 28, it:200/938, loss:1.9168928861618042\n",
      "epoch: 28, it:300/938, loss:1.871867299079895\n",
      "epoch: 28, it:400/938, loss:1.8408300876617432\n",
      "epoch: 28, it:500/938, loss:1.9632482528686523\n",
      "epoch: 28, it:600/938, loss:1.899461030960083\n",
      "epoch: 28, it:700/938, loss:1.94403076171875\n",
      "epoch: 28, it:800/938, loss:2.0364034175872803\n",
      "epoch: 28, it:900/938, loss:2.066302537918091\n",
      "epoch: 29, it:0/938, loss:1.9114816188812256\n",
      "epoch: 29, it:100/938, loss:1.8129019737243652\n",
      "epoch: 29, it:200/938, loss:1.8826686143875122\n",
      "epoch: 29, it:300/938, loss:1.8673946857452393\n",
      "epoch: 29, it:400/938, loss:1.9300293922424316\n",
      "epoch: 29, it:500/938, loss:1.9633452892303467\n",
      "epoch: 29, it:600/938, loss:1.877583384513855\n",
      "epoch: 29, it:700/938, loss:1.9436829090118408\n",
      "epoch: 29, it:800/938, loss:1.9510748386383057\n",
      "epoch: 29, it:900/938, loss:1.9706004858016968\n",
      "epoch: 30, it:0/938, loss:1.829715609550476\n",
      "epoch: 30, it:100/938, loss:1.8968796730041504\n",
      "epoch: 30, it:200/938, loss:1.7518001794815063\n",
      "epoch: 30, it:300/938, loss:1.8946797847747803\n",
      "epoch: 30, it:400/938, loss:1.9108930826187134\n",
      "epoch: 30, it:500/938, loss:1.885815978050232\n",
      "epoch: 30, it:600/938, loss:1.870271921157837\n",
      "epoch: 30, it:700/938, loss:2.020833969116211\n",
      "epoch: 30, it:800/938, loss:1.9392144680023193\n",
      "epoch: 30, it:900/938, loss:1.7939485311508179\n",
      "epoch: 31, it:0/938, loss:1.7955811023712158\n",
      "epoch: 31, it:100/938, loss:1.777928352355957\n",
      "epoch: 31, it:200/938, loss:1.8946202993392944\n",
      "epoch: 31, it:300/938, loss:1.7355067729949951\n",
      "epoch: 31, it:400/938, loss:1.7703197002410889\n",
      "epoch: 31, it:500/938, loss:1.789458155632019\n",
      "epoch: 31, it:600/938, loss:1.8244420289993286\n",
      "epoch: 31, it:700/938, loss:1.8324625492095947\n",
      "epoch: 31, it:800/938, loss:1.9937567710876465\n",
      "epoch: 31, it:900/938, loss:1.9106093645095825\n",
      "epoch: 32, it:0/938, loss:1.7588465213775635\n",
      "epoch: 32, it:100/938, loss:1.7031744718551636\n",
      "epoch: 32, it:200/938, loss:1.766878604888916\n",
      "epoch: 32, it:300/938, loss:1.7574046850204468\n",
      "epoch: 32, it:400/938, loss:1.8694484233856201\n",
      "epoch: 32, it:500/938, loss:1.768075704574585\n",
      "epoch: 32, it:600/938, loss:1.7533469200134277\n",
      "epoch: 32, it:700/938, loss:1.9586759805679321\n",
      "epoch: 32, it:800/938, loss:1.7702200412750244\n",
      "epoch: 32, it:900/938, loss:2.007254123687744\n",
      "epoch: 33, it:0/938, loss:1.8384779691696167\n",
      "epoch: 33, it:100/938, loss:1.6346765756607056\n",
      "epoch: 33, it:200/938, loss:1.7205510139465332\n",
      "epoch: 33, it:300/938, loss:1.6965922117233276\n",
      "epoch: 33, it:400/938, loss:1.6785125732421875\n",
      "epoch: 33, it:500/938, loss:1.8671371936798096\n",
      "epoch: 33, it:600/938, loss:1.7671492099761963\n",
      "epoch: 33, it:700/938, loss:1.7490483522415161\n",
      "epoch: 33, it:800/938, loss:1.8758236169815063\n",
      "epoch: 33, it:900/938, loss:1.8075240850448608\n",
      "epoch: 34, it:0/938, loss:1.7506053447723389\n",
      "epoch: 34, it:100/938, loss:1.636976957321167\n",
      "epoch: 34, it:200/938, loss:1.7440370321273804\n",
      "epoch: 34, it:300/938, loss:1.722664475440979\n",
      "epoch: 34, it:400/938, loss:1.6658258438110352\n",
      "epoch: 34, it:500/938, loss:1.712383508682251\n",
      "epoch: 34, it:600/938, loss:1.7539094686508179\n",
      "epoch: 34, it:700/938, loss:1.6995600461959839\n",
      "epoch: 34, it:800/938, loss:1.7352641820907593\n",
      "epoch: 34, it:900/938, loss:1.8321304321289062\n",
      "epoch: 35, it:0/938, loss:1.7747033834457397\n",
      "epoch: 35, it:100/938, loss:1.7985864877700806\n",
      "epoch: 35, it:200/938, loss:1.747390866279602\n",
      "epoch: 35, it:300/938, loss:1.6150193214416504\n",
      "epoch: 35, it:400/938, loss:1.6618772745132446\n",
      "epoch: 35, it:500/938, loss:1.6157342195510864\n",
      "epoch: 35, it:600/938, loss:1.729142189025879\n",
      "epoch: 35, it:700/938, loss:1.7513062953948975\n",
      "epoch: 35, it:800/938, loss:1.6766486167907715\n",
      "epoch: 35, it:900/938, loss:1.8886874914169312\n",
      "epoch: 36, it:0/938, loss:1.7244505882263184\n",
      "epoch: 36, it:100/938, loss:1.576414942741394\n",
      "epoch: 36, it:200/938, loss:1.6116602420806885\n",
      "epoch: 36, it:300/938, loss:1.790996789932251\n",
      "epoch: 36, it:400/938, loss:1.6994314193725586\n",
      "epoch: 36, it:500/938, loss:1.844691276550293\n",
      "epoch: 36, it:600/938, loss:1.7594045400619507\n",
      "epoch: 36, it:700/938, loss:1.7527945041656494\n",
      "epoch: 36, it:800/938, loss:1.6841237545013428\n",
      "epoch: 36, it:900/938, loss:1.6741052865982056\n",
      "epoch: 37, it:0/938, loss:1.6105138063430786\n",
      "epoch: 37, it:100/938, loss:1.7117077112197876\n",
      "epoch: 37, it:200/938, loss:1.573431372642517\n",
      "epoch: 37, it:300/938, loss:1.6989872455596924\n",
      "epoch: 37, it:400/938, loss:1.640057921409607\n",
      "epoch: 37, it:500/938, loss:1.7235076427459717\n",
      "epoch: 37, it:600/938, loss:1.68967604637146\n",
      "epoch: 37, it:700/938, loss:1.6866928339004517\n",
      "epoch: 37, it:800/938, loss:1.7668561935424805\n",
      "epoch: 37, it:900/938, loss:1.7246088981628418\n",
      "epoch: 38, it:0/938, loss:1.6472492218017578\n",
      "epoch: 38, it:100/938, loss:1.6075068712234497\n",
      "epoch: 38, it:200/938, loss:1.6459300518035889\n",
      "epoch: 38, it:300/938, loss:1.6011794805526733\n",
      "epoch: 38, it:400/938, loss:1.650064468383789\n",
      "epoch: 38, it:500/938, loss:1.5825331211090088\n",
      "epoch: 38, it:600/938, loss:1.5992945432662964\n",
      "epoch: 38, it:700/938, loss:1.5733428001403809\n",
      "epoch: 38, it:800/938, loss:1.683924913406372\n",
      "epoch: 38, it:900/938, loss:1.6396493911743164\n",
      "epoch: 39, it:0/938, loss:1.4940974712371826\n",
      "epoch: 39, it:100/938, loss:1.551439881324768\n",
      "epoch: 39, it:200/938, loss:1.6238789558410645\n",
      "epoch: 39, it:300/938, loss:1.5342535972595215\n",
      "epoch: 39, it:400/938, loss:1.643222451210022\n",
      "epoch: 39, it:500/938, loss:1.5746535062789917\n",
      "epoch: 39, it:600/938, loss:1.706485629081726\n",
      "epoch: 39, it:700/938, loss:1.6514743566513062\n",
      "epoch: 39, it:800/938, loss:1.657994270324707\n",
      "epoch: 39, it:900/938, loss:1.6344592571258545\n",
      "epoch: 40, it:0/938, loss:1.6624133586883545\n",
      "epoch: 40, it:100/938, loss:1.5424904823303223\n",
      "epoch: 40, it:200/938, loss:1.5752809047698975\n",
      "epoch: 40, it:300/938, loss:1.5808141231536865\n",
      "epoch: 40, it:400/938, loss:1.6330569982528687\n",
      "epoch: 40, it:500/938, loss:1.5986095666885376\n",
      "epoch: 40, it:600/938, loss:1.5901556015014648\n",
      "epoch: 40, it:700/938, loss:1.661420226097107\n",
      "epoch: 40, it:800/938, loss:1.7710168361663818\n",
      "epoch: 40, it:900/938, loss:1.6185319423675537\n",
      "epoch: 41, it:0/938, loss:1.628125548362732\n",
      "epoch: 41, it:100/938, loss:1.5950627326965332\n",
      "epoch: 41, it:200/938, loss:1.4311665296554565\n",
      "epoch: 41, it:300/938, loss:1.7013697624206543\n",
      "epoch: 41, it:400/938, loss:1.5173938274383545\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-66fbaa2b6c52>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcriteria\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Backward & update weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m         \u001b[0;31m#print(model_prior.layers[0].hconv.weight.grad[0])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0;31m#print(z_float.grad[0].mean())\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    182\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m         \"\"\"\n\u001b[0;32m--> 184\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    123\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    124\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model_vqvae.eval()\n",
    "optimizer = Adam(model_prior.parameters(), lr=lr, betas=(0, 0.99))\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.2, patience=5, verbose=True, threshold=1e-4)\n",
    "criteria=nn.CrossEntropyLoss()\n",
    "\n",
    "\n",
    "# images,_,_=next(iter(train_loader))\n",
    "# images=images.cuda()\n",
    "# _, _, _=model(images)\n",
    "# shape=model.ze.shape\n",
    "# z=model.indices\n",
    "# z=z.reshape(shape[0],shape[2], shape[3]).detach()\n",
    "\n",
    "# z_float=(z.float()/(Klevels-1)).unsqueeze(1)\n",
    "# z_float.requires_grad_()\n",
    "\n",
    "for epoch in range(nepochs):\n",
    "    for it,(images, _,) in enumerate(train_loader):\n",
    "        \n",
    "        images=images.cuda()\n",
    "        _, _, _=model_vqvae(images)\n",
    "        shape=model_vqvae.ze.shape\n",
    "        z=model_vqvae.indices\n",
    "        z=z.reshape(shape[0],shape[2], shape[3])\n",
    "\n",
    "        z_float=(z.float()/(Klevels-1)).unsqueeze(1)\n",
    "        #z_float.requires_grad_()\n",
    "        #print(z.min(), z.max())\n",
    "\n",
    "        #imgs,imgs_quant= discretize_imgs(images, Klevels)\n",
    "        #imgs=imgs.cuda()\n",
    "        #imgs_\"quant=imgs_quant.cuda()\n",
    "        #print(z_float.shape)\n",
    "        logits=model_prior(z_float.detach())\n",
    "        #print(logits.shape)\n",
    "        #print(z.shape)\n",
    "\n",
    "        loss=criteria(logits,z)\n",
    "        optimizer.zero_grad() # Backward & update weights\n",
    "        loss.backward()\n",
    "        #print(model_prior.layers[0].hconv.weight.grad[0])\n",
    "        #print(z_float.grad[0].mean())\n",
    "        #print(z_float[0])\n",
    "        optimizer.step()\n",
    "\n",
    "        if it%show_every==0:\n",
    "            #print(z_float.grad[0].mean())\n",
    "            print(f'epoch: {epoch}, it:{it}/{len(train_loader)}, loss:{loss.item()}')\n",
    "            \n",
    "    scheduler.step(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 1, 28, 28])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAATtklEQVR4nO3dbYyV5ZkH8P+fN3kZEEZkQCDbStBI1ixsJmQTN0ZDtrFKxCaygQ8NJmSnH2rSJv2gYT/Uj8Zs29RkbUJXUrqp1prWQCLZrWITxcTCaFhBcZU10E6ZzGB4G5AXh7n2wzw2U5znuo7Pfc5zDr3/v4TMcK65z7nPc+aa83I9933RzCAif/2mtHsCIlIPJbtIJpTsIplQsotkQskukolpdd7YlClTbOrUqaXxlMoAycpjG7ltL55622NjY2582jT/YVq4cGFp7MYbb3THXr161Y1Hx2XKFP/5YmRkpDQ2PDzsjo2kHndP9JhEtx3FW1UFGxsbw9jY2KQ3npTsJO8D8GMAUwH8h5k96f381KlTsWDBgtL4Z599Ft1eaWzGjBnu2Eh026Ojo6Ux7w8YED+wly9fduPeMQOArVu3lsYeeOABd+zp06fdeDT3G264wY2/8cYbpbGnn37aHZuaUCl/DKLHZPr06W48+p3w/shGf2g8586dK41VfhlPciqAfwfwdQCrAGwmuarq9YlIa6W8Z18L4KiZfWxmVwD8EsCG5kxLRJotJdmXAvjjhP8PFJf9BZJ9JPtJ9qe8PBGRNCnJPtkboi+8wTOz7WbWa2a90Yc5ItI6Kdk3AGD5hP8vA3AibToi0iopyX4AwEqSXyU5A8AmALubMy0RabbKpTczGyX5KID/xnjpbYeZvReMcUsOUZnHK6VEnwdcunTJjc+ZM8eNe3OLynYzZ85047fffrsbf+SRR9z4+vXrS2NRaez8+fNuPDJ37lw3fuedd5bGZs2a5Y598cUX3fjx48fd+CeffFIaix6T2bNnu/HouEa/b57UcxvKJNXZzWwPgD0p1yEi9dAnZiKZULKLZELJLpIJJbtIJpTsIplQsotkgnXuLjt9+nTzlmtGywq9Zaypp+JGt7106RdO+/+zqA6+bt06N75s2TI3HtV8BwYGSmPe0lwgvt9RPbmrq8uNe/Xmnp4ed2y0tLe/v9+NP/PMM6WxXbt2uWO9dfhAPPeozn7lypXSWJSTXh6cOXMGo6Ojk56Qomd2kUwo2UUyoWQXyYSSXSQTSnaRTCjZRTJRa+lt2rRpFm1t7PF29EzZChqIt2ves6d8cV9Uhjl79qwbj8o0UVnRW94bLeWMSkzRcYnKgl5pz9sJFYjLeosWLXLj3uPy1FNPuWO3bdvmxru7u914VLL0Sm8pVHoTESW7SC6U7CKZULKLZELJLpIJJbtIJpTsIpmotWUzSbe7ZVR79LZsTu3o+eCDD7rxW265pTQWtR6OavzRMtSoY6hXp4+WsF64cMGNR1t0pyyhje5XdH7C0NCQGz958mRpbNOmTe7Y/fv3u/GXX37Zjc+fP9+Np9TZq7ZR0zO7SCaU7CKZULKLZELJLpIJJbtIJpTsIplQsotkovb17F6LX6+dcxSPWgdH67KjrYVXrlxZGovWZUfxqBaeUuuO2km3mrcWP2oXHdXho3X+3jkdN998szv2ww8/dOMbN25041E7au93OWof7j2m3nr2pJNqSB4DMALgKoBRM+tNuT4RaZ1mnEF3r5mVd70XkY6g9+wimUhNdgPwW5Jvk+yb7AdI9pHsJ9lf9ZxeEUmX+jL+LjM7QXIRgFdIfmBmr0/8ATPbDmA7MP4BXeLtiUhFSc/sZnai+DoM4CUAa5sxKRFpvsrJTnIOybmffw/gawAON2tiItJcKS/jewC8VKwTnwbgOTP7r2hQynp2bw/0qJa9fv16N37bbbe5cW9tdbSm+9SpU2482ts94q3Vj84viObutQcG4lq395hGdfQontLiOzqnY+1a/0Xqvffe68ZfffVVNz5v3jw37vEeb/d3oeoNmtnHAP6u6ngRqZdKbyKZULKLZELJLpIJJbtIJpTsIpmofStpr1QTlXE80am4Dz/8sBuPymPeds1Ry2Wv3AjE5a1oGbJXRoq20I5aC0dzj5bQeuOj0lrUDjpaRurdt+h+Ra3F7777bje+b98+N+6J8iB6TEuvt9IoEbnuKNlFMqFkF8mEkl0kE0p2kUwo2UUyoWQXyUStdXbArxmntF1etGiRO/bWW29144ODg2584cKFpbGRkRF3bFQ3jZZqRrXwlO2+oqWe0dxTtiJP2Tq8kdv2zl+IlhWfPn3aja9atcqNd3d3u/GLFy+WxqL7VfWY65ldJBNKdpFMKNlFMqFkF8mEkl0kE0p2kUwo2UUyUWud3czcmnBUV/XWTt9xxx3u2Gjr3qh9sNdG9/3333fHRmufL1y44Maj9fKe1JbcKXsMRLcf1bqj8wui++aNj+5XtDV51PJ58eLFbvzo0aOlsdRjXnq9LblWEek4SnaRTCjZRTKhZBfJhJJdJBNKdpFMKNlFMtFR69mjuqm31/eaNWvcsVHr4qi2eebMmdLYgQMH3LFR+99o//SolbU39+iYRmvho+MWjffi0f2K9oWP7pt3Xka0B0F07sPx48fdeDR375yS6PyD6HyUMuEzO8kdJIdJHp5wWTfJV0h+VHxdUOnWRaQ2jbyM/xmA+6657HEAe81sJYC9xf9FpIOFyW5mrwO4tjfSBgA7i+93AnioyfMSkSar+p69x8wGAcDMBkmWbgBHsg9AH9C6c35FJNby7DOz7WbWa2a9VRvSiUi6qsk+RHIJABRfh5s3JRFpharJvhvAluL7LQB2NWc6ItIq4Xt2ks8DuAfAQpIDAL4P4EkAvyK5FcAfAGxs5MZIurXyqCe2V1f94IMP3LEvvPCCG1+xYoUb9+rFhw4dcsdG67Kjmm5Uh/fqrlEf8igeGR0ddePe5zRRDT+qw3d1dbnxt956qzT23HPPuWOjOnwk2h9h7ty5pbHouFTdcz5MdjPbXBJaF40Vkc6hj8dFMqFkF8mEkl0kE0p2kUwo2UUyUfsSV6/U4y1JBPwS05tvvumO3b9/vxuPbtvbSnpoaMgdO3/+/KTbjso4XmkvOmsxWqIaLSNNaekcldZS20l7S49fe+01d2xk9uzZbjx6TL1yaur9Lh1XaZSIXHeU7CKZULKLZELJLpIJJbtIJpTsIplQsotkovaWzV774Wi5pVdfTKn3AvFSTa9evWzZMnfsTTfd5Maj9sDRksfLly+7cU9qS+eoju9df7TdclSHj5ZEnz59ujSW2i46+n2ZMWOGG/d+X6OxUQ2/9DYrjRKR646SXSQTSnaRTCjZRTKhZBfJhJJdJBNKdpFMdNR69mgdb7SlsieqTUbxTz/9tDS2ePFid2xUL/baQQNp7aajOnhql56Uls1RrTr6fYiOq1fjj+539PuQWmf3zjeJHm8v7t0vPbOLZELJLpIJJbtIJpTsIplQsotkQskukgklu0gmaq+zu3XAYE26F49qslHtMqrhezXbaD15VEeP7nc0N6+WHV13tIdAtN49Ou7e9afe76gW7rV0jmr00Xr21FbXXh0+Oi5Vz40In9lJ7iA5TPLwhMueIPknkgeLf/dXunURqU0jL+N/BuC+SS7/kZmtLv7tae60RKTZwmQ3s9cBnKphLiLSQikf0D1K8t3iZf6Csh8i2Ueyn2R/6n5nIlJd1WT/CYAVAFYDGATwg7IfNLPtZtZrZr2piy5EpLpKyW5mQ2Z21czGAPwUwNrmTktEmq1SspNcMuG/3wBwuOxnRaQzhHV2ks8DuAfAQpIDAL4P4B6SqwEYgGMAvtXIjZGs3Fsa8OvJ0ecBqX3GvZqut9YdiGu6kWjttFePjt46RXXySEr/9+h+pfyuAMC8efNKY9E6/NS3nNHcvd+J6PyBaM/7MmGym9nmSS5+ttKtiUjb6HRZkUwo2UUyoWQXyYSSXSQTSnaRTNTestkr9URLGr1STTQ2Kn9FJSivFBOVaaLlkNHco+W5Kachp57CnLIcMzou0XGNjsuCBaVncSe1VG4kHi179u57dL+8PPAeTz2zi2RCyS6SCSW7SCaU7CKZULKLZELJLpIJJbtIJmqts5N064vR0j6v/hgtl0xtXezFU9oWNyJlfMoS1EbGR3GvHh093qnbNXvnL0TnVUTnH0Rzj+rw3u2nnn9QOqdKo0TkuqNkF8mEkl0kE0p2kUwo2UUyoWQXyYSSXSQTtbdsdtfbBrXJlNbEUT05ZdviaK18q2vZKcc0qummHteUuUVS2i5HdfTUraRTWmFHY6tu/61ndpFMKNlFMqFkF8mEkl0kE0p2kUwo2UUyoWQXyUTt+8Z7ddmU9sKpNdvotr31yym1ZiB9bbW3zj91bqn7ynvjU9tFR7w6e7Q3e1TDj/b6Tzl3Ilor37L17CSXk/wdySMk3yP5neLybpKvkPyo+Fq+I7+ItF0jT4ejAL5nZncA+AcA3ya5CsDjAPaa2UoAe4v/i0iHCpPdzAbN7J3i+xEARwAsBbABwM7ix3YCeKhVkxSRdF/qPTvJrwBYA+D3AHrMbBAY/4NAclHJmD4AfUD6+2oRqa7h7CPZBeDXAL5rZucaHWdm282s18x6UxcXiEh1DSU7yekYT/RfmNlviouHSC4p4ksADLdmiiLSDOHLeI4/HT8L4IiZ/XBCaDeALQCeLL7uauC6kl7Ke+WOaFlgVEqJeC1+o22sU7cGjo6Zd/vRdaduwR3xykgp7Z6BuHSX0rI5KllGj2lUPktZ+lu1HNrIe/a7AHwTwCGSB4vLtmE8yX9FciuAPwDYWGkGIlKLMNnNbB+Asj+x65o7HRFpFX08LpIJJbtIJpTsIplQsotkQskukonat5L2aqdR3dSrs0d1zdRauFd3nTVrljs2ktry2avLpl53K1s2p7ayjmrlXp09mvfFixfdeFdXlxuPtGpLdY+e2UUyoWQXyYSSXSQTSnaRTCjZRTKhZBfJhJJdJBMdtZV0VOv2auUpdXIgrZ4c1UXPnDnjxi9fvuzGZ8+e7cZT1oxHa6NT122n1NK9raAB4NKlS2583rx5pbHouMycOdONp7RkjqS06PbomV0kE0p2kUwo2UUyoWQXyYSSXSQTSnaRTCjZRTJR+3p2r0YY1Q+9+mNqHT2ltXHqnvXRbUfjU1o2p+7NHt13b3w0t6iOHrVd9s7LmDNnjjv27Nmzbjz1vA4vHh3zlrVsFpG/Dkp2kUwo2UUyoWQXyYSSXSQTSnaRTCjZRTLRSH/25QB+DmAxgDEA283sxySfAPAvAE4WP7rNzPY0cH2lsZR+3an9tKPxXk03Wm8enT8Q1Yujuqs395Te7kDaumzAP0cgOn8gmnsU9/bz9/aUB4Dz58+78Wgdf8o+AqnnPpRp5KSaUQDfM7N3SM4F8DbJV4rYj8zs3yrdsojUqpH+7IMABovvR0geAbC01RMTkeb6Uu/ZSX4FwBoAvy8uepTkuyR3kJz0dRHJPpL9JPtTXxKKSHUNJzvJLgC/BvBdMzsH4CcAVgBYjfFn/h9MNs7MtptZr5n1Vj2nV0TSNZTsJKdjPNF/YWa/AQAzGzKzq2Y2BuCnANa2bpoikipMdo4/HT8L4IiZ/XDC5Usm/Ng3ABxu/vREpFka+TT+LgDfBHCI5MHism0ANpNcDcAAHAPwrdTJpJTPos8DolKJ1w46Em0FHZWYenp63Hi0HNM7LlEJKJp7dFyisqF33EdGRtyxKaVYwJ9b1HI5+n1J/fzJu29RaS06LmUa+TR+H4DJjmpYUxeRzqEz6EQyoWQXyYSSXSQTSnaRTCjZRTKhZBfJRK1bSZN0a4RR3dSrfUZjo9pkVFf1atknTpxwxz722GNuPKqzz58/3417ddmoTp7amjg6h+DixYulsahWHd3v6DH1tqI+fvy4OzalhTcQ3zdv7qlttkuvt9IoEbnuKNlFMqFkF8mEkl0kE0p2kUwo2UUyoWQXyQTr3BeO5EkAEwucCwF8UtsEvpxOnVunzgvQ3Kpq5tz+xsxunixQa7J/4cbHN6HsbdsEHJ06t06dF6C5VVXX3PQyXiQTSnaRTLQ72be3+fY9nTq3Tp0XoLlVVcvc2vqeXUTq0+5ndhGpiZJdJBNtSXaS95H8X5JHST7ejjmUIXmM5CGSB0n2t3kuO0gOkzw84bJukq+Q/Kj46vcernduT5D8U3HsDpK8v01zW07ydySPkHyP5HeKy9t67Jx51XLcan/PTnIqgA8B/BOAAQAHAGw2s/drnUgJkscA9JpZ20/AIHk3gPMAfm5mf1tc9hSAU2b2ZPGHcoGZ+btj1De3JwCcb3cb76Jb0ZKJbcYBPATgEbTx2Dnz+mfUcNza8cy+FsBRM/vYzK4A+CWADW2YR8czs9cBnLrm4g0Adhbf78T4L0vtSubWEcxs0MzeKb4fAfB5m/G2HjtnXrVoR7IvBfDHCf8fQGf1ezcAvyX5Nsm+dk9mEj1mNgiM//IAWNTm+VwrbONdp2vajHfMsavS/jxVO5J9ss27Oqn+d5eZ/T2ArwP4dvFyVRrTUBvvukzSZrwjVG1/nqodyT4AYPmE/y8D4O/YWCMzO1F8HQbwEjqvFfXQ5x10i6/DbZ7Pn3VSG+/J2oyjA45dO9uftyPZDwBYSfKrJGcA2ARgdxvm8QUk5xQfnIDkHABfQ+e1ot4NYEvx/RYAu9o4l7/QKW28y9qMo83Hru3tz82s9n8A7sf4J/L/B+Bf2zGHknndCuB/in/vtXtuAJ7H+Mu6zzD+imgrgJsA7AXwUfG1u4Pm9p8ADgF4F+OJtaRNc/tHjL81fBfAweLf/e0+ds68ajluOl1WJBM6g04kE0p2kUwo2UUyoWQXyYSSXSQTSnaRTCjZRTLx/9wHtMs9RVIsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "gens=generate_imgs(model_prior, shape=(shape[2], shape[3]), nimgs=16)\n",
    "gens=(gens*(Klevels-1)).long().squeeze()\n",
    "out=model_vqvae.compute_out_from_indices(gens)\n",
    "\n",
    "plt.imshow(out[0,0].detach().cpu(),cmap='gray')\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAJACAYAAACdeiLBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdefBdZ33n+e9pjDfJsrXvsmRLsrEtC+OVgW5MjAPBEEHTSSeVxVB0uYuaMD1ToQeS6kr/1VV0Znpqqno6UFQmEyZD2k1DJzjQneCwBDs48b7ItmzJsnZZm7V6Y8mZP1Bm9Hyej8/5cnTvb5Hfr3/Ec/3cc8/ynHMPv/O536dp2zYAAADg/YPJXgEAAICpjJslAACADtwsAQAAdOBmCQAAoAM3SwAAAB24WQIAAOgwtpulpmne1zTNM03TbGma5jPj+hwAAIBxasZRZ6lpmjdFxLMRcWtE7IqIByLil9u2fep1+lPsCQAATLSDbdvO7+s0rr8s3RARW9q23dq27Q8i4s6I2DCmzwIAABhie6bTuG6WlkbEzlPau06+9v9pmuaOpmkebJrmwTGtAwAAwGk7a0zLbcxrxaO2tm2/EBFfiOAxHAAAmLrG9ZelXRGx/JT2sojYM6bPAgAAGJtx3Sw9EBFrmqZZ1TTN2RHxSxFx15g+CwAAYGzG8hiubdsfNU3zGxHxFxHxpoj4g7ZtnxzHZwEAAIzTWEoH/NQrQWYJAABMvIfatr2urxMVvAEAADqM69dwI9c05Q/sMn8Re9Ob3tTb58c//vHgdepy9tlnV6/9s3/2z4r2bbfdVvU5cuRI0f67v/u7qs8555xTtO+5556qz7//9/8+tZ6n+gf/oL531v2c2e+6HLcNbxRLliwp2itWrKj63HDDDUV77ty5VR8dT+eff37vZ7/22mtFe+vWrVUfHTtPPjm5T8sZO5hM5557bvWaXvNcn6uuuqpov/e97y3aO3bsqN7z0ksvFW39jouImDNnTtF+85vfXPVZvHhx0f7Rj35U9dHzSK8NEREHDx4s2l//+teL9q5du6r3zJ9f1nJ8+eWXqz4nTpyoXlO67e67W4/DuL67Xw9/WQIAAOjAzRIAAEAHbpYAAAA6TJvM0pBf7bm8gy7nrLPqXeCe+fZZvXp10f7oRz9a9fngBz9YtDV7FJF7vnvBBRcU7auvvrrqc9555xXt//yf/3PVZ/v2ckqcIfkQl836wQ9+8FMvZ6rJZOQ0P/DDH/6w6vObv/mbRfsd73hH1UfHoDsOumyXb9Dn/JpL+NCHPlS9Z82aNZ3r6z5rnL+gJaME5fIrQ/Iq7pzRsTz02nXzzTcX7WuuuaZo7969u3rPggULivb69eurPjNmzCjammGKiHj11VeLtsuevvLKK0Xb5SJ1ObpvNMMUUR8b9/01a9asou2Ow9GjR4t25jt4VN/dWfxlCQAAoAM3SwAAAB24WQIAAOjAzRIAAECHaRPwHhIydUEyfc0FwjR497GPfazqc8sttxTtZcuWFW0N5kVE7Ny5s2hrAcqIOpTrQuC6DRrei4i44447ivanP/3pqs8DDzxQtH/v936v6nPXXd3zH7sw4ZlgSIhZw9IREW95y1uKthZ+i6j3YaaYqguTuzF3qj179lSvaUE7dzz1tXGGKDF+ExnYV0OKjo6q+GBmO11oWM9Hd+7pObF///6i/eCDD1bvmT17dtHWHwlF1Nf/vXv3Vn30hyaZa7IGqiMiXnjhhaJ98cUXF+1f/dVfrd7z/PPPF+3jx49XffQa8/jjj1d99DVXNFNN9HXozPymAwAAGBFulgAAADpwswQAANBh2mSWhkyc696jr7mJCf/4j/+4aC9atKjqo8989Vmye26sz+fdhIw6EaErkqbFw1yfjRs3Fm0tZBkRsXTp0qL9+7//+1Wff/tv/23R/nf/7t91rktErljjqIwqy6brnClOp9t10003VX30mLtMRGaiTv2smTNnVn0036DLPXbsWPUenZBXC8hF+Gyd0nPPFSvV7MmZULx0qhmSCcoUaxyVIUVHR7V+Q4tS6lh21/ZVq1YVbT2vLrnkkuo9eq65czpzXul2uZyrnnsuF6mFjHVfuO3WfK8rSqnfn+95z3uqPp/61KeKtiviOdn4yxIAAEAHbpYAAAA6cLMEAADQgZslAACADtMm4J0J8g4pXnbbbbdVry1ZsqRoHzhwoOqjn68FslxwXMPQLhytAW8XiNTPcgFqDQK6kK4WIdPiYRERv/zLv1y0//Zv/7Zo33vvvdV7NCg4zoD3kKCne8+QsPE//af/tGj/43/8j6s+GqJ3Y/Tw4cO966IFJ10h0r5gvdtuDf5v2LCh6vPEE08U7Ycffrjqo9uVKdyH0csEqCeyCKUa8oOMyVzfiHqfuqC4FpjUkPWKFSuq92ig2/24Qr300kvVa5milHruuR+aKL2Ou/NXr0vuxyn6wxIt1BsR8da3vrVoE/AGAACYZrhZAgAA6MDNEgAAQIdpk1ka8tzaFd7SZ7Wf+MQnqj76rPaiiy6q+mhRyiGTjbo+LuukdJJB94xa188tV7fTFS3UZ9Af//jHi7bLLLnlKD0O48yzaMbAZQ76sh7vfve7q9d0smJXVE7zSC6z9OijjxZtVwR13rx5RTtTPE/3qcsTaJ8PfvCDVZ93vetdRfuLX/xi1eev/uqvOpc71bgxoCY7KzOZRnHOOKPap6PKPum1022nfo+481Ozf3p+rl27tnqPno+ZrFGmKGUmf+nysnpt0mv0nDlzqvdoNnHhwoVVHx0n7nv5sssuK9rf+MY3qj7KLWdUky47/GUJAACgAzdLAAAAHbhZAgAA6MDNEgAAQIdpE/AewoW9NKyqwbKIOhztCuxpkUcN3rmgoL6WCahlAn1u/TSc5wLeGhTXmbIj6oDhjTfeWLSvv/766j0PPPBA9dpU4vZ7X1j1Yx/7WPWa7lNXSE2DlNu2bav6PPLII0X74osvrvpce+21nZ+d4QqTashfx0REXazUFZXTgLej+32cYcw+Exnezsx2P9Vo2Ngdq8k8nqPaf3reZ66Ty5cvr/roMdbrthZvjKj3VyaA7sLbLhiuMtuphTQzAf5MMVr9LHf91etbxkRfP/jLEgAAQAdulgAAADpwswQAANBh2mSWhhQhc/7JP/knRfvFF1+s+mQmvNX10ee9mefPGW7CQ30u7Jarz4k1mxJRF0VzRQv18y+88MKi/Y/+0T+q3pPJLGWKwWWOcWZc6Gvuubquj05q6XJEO3bsKNruWbxOsHn33XdXfZ599tmirfs4IuL5558v2pdeemnVRyfZ1PVxY0AzEDqRc0Sds3KT+GbyK3qOuM/qmwx4OnJjUsetjj93TmfO+yGFIp3McjK5pr5snds3mUnJJ3Jc6Ha6c08zSno+uKyRnq9uO/V9mayi26eZ81PHpF77MwU73WTAWszy+PHjVR+XHe5DUUoAAIAphJslAACADtwsAQAAdOBmCQAAoMO0CXgPCXS7QOsll1xStPfu3Vv1mTt3btE+ceJE1UfDbhoC1zBrxLAiXy6wllmO9nHvyRQq0314+PDhon3FFVdU79Hili4QrMdzsov06effcMMNRTuzDTNnzqz6aODR9dHlZGYEd2NSl6PBSjcmdbb0nTt3Vn22bNlStB966KHe9XNcIFOdCYHuDD1WmQKFGrjVILTjfrThxpe6+uqri7YruKrhY3fs+kLpmWvZZIa5I+r9vHbt2qqPhrX1PHfHSrfLbaeOC7e/dJ3dZ7lwttIxp+er+7GRjgF3ndRAt/tRle6vNWvWVH02b95ctClKCQAAMIVwswQAANCBmyUAAIAO0yazNITL02iGxGU/9Pnpt7/97aqPTsiry8kUsswUUHTP0DPP/TWfkslQuT76WceOHSva8+fPr96zaNGior1169bezx5qVFknXY5OEOwmmNVsgNt/un/chJqZjNcLL7zQ20fHreYJVq1aVb1HJ9e9//77qz5f+9rXirbLHCiXf8tkZaaSoYVSR1FYM5MxdPR8dTmTn//5ny/aH//4x6s+WjD0k5/8ZNVHj6fLR+kY7Jtw1r1nIrljrrlNN5GuXgv0XHz66aer9+j+cmNL94XL/en4clket59VX2bJfbbmIvfv31/10YK+bsJ2Ld7rvrs1s+QMycRl8ZclAACADtwsAQAAdOBmCQAAoMMZnVm65pprqtc0T+AyQVpLyOU4tA6PPhPWuhsRwzJLro8+h3V99Dl2Jmvh6nNoFku3a/v27dV73DNppVmKia6Z0UdzV+54XnTRRUXb5UM0r5WpQ+KyAfr57ljpuNB8mctU6XIffPDBqk8mo6TrPJn1koZmjUbxnoiJG8tLly6tXnv7299etNevX1/1WblyZdF227lnz57ePplskY45HRduGVr7y028Oi5ufTQ/4/a7rqNeA/ft21e9R2v+uXyNjuVMbS23nMz3Sl/ex13X9fjqd2dEnYvUsRUR8Sd/8idFO5NPckaZUVL8ZQkAAKADN0sAAAAduFkCAADowM0SAABAh2kb8HZhWg2tuUJgd955Z9G+9NJLe5ezcePGqo8WFNOgrCvKp+E8tw0ZGiB1IXVdtgsuauG0v/mbv6n6fOlLXyraGvh2gTotaOcmNNZAsCv6OJE0SLl48eKi7YqXauBRA9UREc8880zvZ19++eVFe968eVUfDUm68aXjQosGHj16tHqPBm63bdtW9dHt1OPrPtvJ/MBhFIYud1QF7fR9riCg7nc9ni5M+/73v79o33rrrVWfvuuS+ywXytV94Qqa6ja465me17pvtBhhRMT73ve+ou0Cwd/5zneq14bIHHO9brvvg7e+9a1FW4sfuwnbtViju37oPnU//tDx7q4NGfo+/Ww3lq688sqi/eUvf7nq87nPfa5ou+9l5ca/nkcTXbyUvywBAAB04GYJAACgAzdLAAAAHaZNZkmf1WaKc/31X/919ZoWmHTF8zTL4/IqWpBQn5+692gBL1c8L1NwMpP90PVxmRJ9Xu+Kb7rXppJR5Uw0Y6DPxzOTFbt8kubAVq9eXfXR4nQu46XH2PXRzIGOEzfWNVOi2Si3XGdIwdVxGTo5aybnN4Rbjl7PtJiktiPq3JAbb3ocLrvssqqP5vHc/tLX3MSmmzZtKtrf//73qz56HbzwwguLtub1IupCmk899VTVZ1SZpcz1QrfzX/2rf1X10SK2mkdyx2rDhg1F22WWNHfoMkvKbZNe/91ytI8ux71Hv4fd90Umo6TcpNuZY6XncOY+IYu/LAEAAHTgZgkAAKADN0sAAAAduFkCAADoMG0C3kNm8nbFGjM0JKlh7oiIuXPnFm0t+OdCsZnCi5kQm66fe48GgF3IVLdryOzyQ2dYH9VyMqFhHQcuqKhF0PrC0hF1WNUFqHX2bFcEVZfjCrLpsXJjW4+xBtBd2NGFe5Vul9sXExXezhgazB7yPheg1mO8Zs2aqo+eswsWLCjabn/u3r27aLsw/sqVK4u2/lgloh47x48fr/roeNOxFBGxbt26or1w4cKqj55r+tluLOlnuXC5rp/bF+Pi9pe+tnXr1qLtCs3OmTOnaLsfBel1yI0Ld91RmQLIOibdj0j6lrtv377e9zhDCk66bRploFvxlyUAAIAO3CwBAAB04GYJAACgw7TJLI1K5vmuThSqBcci6meqmllyuRh9zT2vV5k+mefY7lnukIzG0GzRuJaTycpkiqvpftYx4J6Pa27tlVdeqfpoRsNlP7Qgpsu7aY7JbYNmHrSPmwBXMy3z58+v+hw4cKBoT6V8kuMyhloA0G3DCy+8ULQ1UxJRFxBdtmxZ1UfPNTeBsU4Oq1kPlzW64IILOtsRdc7Ebaeuj8uZ6FhyY1snVnUT8uq41WxWJnvn9oUud1SZpUyxxgyd0NgdKx1vLjOqWR63vzIFJ5UbF3qd1Iytuy7pNrgJgzMm87soi78sAQAAdOBmCQAAoAM3SwAAAB24WQIAAOhwRgW8MzPQZ4oh9gXdIuqQpAaEXbE/Xa4L6+n6ZWZvd4G+TB8Ngc+YMaPqMy5Dg+t93Hbqfndhdw1ganjV0TCtC2hq0FPD3BERs2fP7l0/DVe6oPiuXbs6l+NmNdcZ6HVdIurtmuhg5U/rxhtvrF775Cc/WbRd6FoLCbo+es66a0MmHK3jX8ef+0GBzsTuArd6Trtt2L59e9F2wWwNVbv10XV2xVT1Nf2xgLtO6nnliiNmiiwOkbnmuP2u54QeB/fDIj2emc92556OJXdt1XHrtqGvgKh7j8r8gCpT1DYztif6hyb8ZQkAAKADN0sAAAAduFkCAADoMG0yS/qcM1NUayhdtsuv9E2K657dDikw5rYzk83S5WQyQjqh61CZY6WvZdYvY2ieRrNF2nbHU9dZJ82N8BNoKs0uuMySZj/cpJuaWXK5JqXLcev73HPP9S4nY1STJ/dx25CZJFoLJrr9p8UPXUYuUzxSj6eewzr+Iurz3o3JzPVDz3OXG+qblNmtj1uOFkLVCWddoVRdrtsXWgx0x44dVR+VycpkcjCZXI7uP1fgVPOBboJezWZlvuPc94puVyb7pGPbHd8h53BmHw+dEHec1xj+sgQAANCBmyUAAIAO3CwBAAB04GYJAACgw7QJeGeKLGZmXx4S+HLBQF22FtHKhOFcWC8TfNYAX2b9XB8ND7qChH1cmHBI0H6yZ7LXfXHw4MGi7WaX12J5hw4dqvqsX7++aLuilHqsXLhR+7ixre/TYHFmTLqQ/6hCk6P6AUafL33pS9VrTz31VNF+z3veU/W5+OKLi/bq1aurPjpO3LHSH4S4Y6UB70xoWD/LFZHtKywYUYe1M+PC/aBF18fNHK+fpeeMC+Nr0FkLZEbUx+r73/9+1UdlrjGuT+aY93Ehfz027rMz1/Eh56MbF33fK2679T3uO0SLnrqConptyFwrRvWjoCz+sgQAANCBmyUAAIAOg2+WmqZZ3jTNd5qmebppmiebpvkXJ1+f0zTN3U3TbD7570//bAcAAGCKOJ3M0o8i4jfbtn24aZoLIuKhpmnujoiPRsS32rb9bNM0n4mIz0TEp093RfX5pHtOO66CVEOKPrrnu5lnrLpc92xZZSbJdX1UZqJENVE5lNeTKYCpXEZDt13bbizpBKluHy9atKhou+J++gzfjRPNuLhjpduemZBUcyZuO4cWiFMTlUtz++aRRx4p2ps2bar6XHrppUXbTci7Zs2aon3FFVdUfebOnVu0XUZDz8/M9U3PNTdO9BhnxpI7HzRb54p6ZiYV1uXoeJs/f371Hi2UqpM9R0Tcfffd1Wt9MkUpnVGMf5dZynx25jsjk2vK5Fz7cnMuI5f5flLT7Xry9wb/Zalt271t2z588n8fj4inI2JpRGyIiC+e7PbFiPjQ6a4kAADAZBlJZqlpmpURcU1E/G1ELGzbdm/ET26oImLB678TAABgajvt0gFN08yMiK9GxP/Ytu2x7M/5mqa5IyLuON3PBwAAGKfT+stS0zRvjp/cKH2pbdv/cvLlfU3TLD753xdHxH733rZtv9C27XVt2153OusAAAAwToP/stT85E9I/2dEPN227f92yn+6KyJuj4jPnvz3a6e1hidlwlx9oevscpQLvvUVj8x8tgsEaxg0U/TRhawzBc8yM4uPi67f0KB4pqBj32dH1Md4//7yHn/t2rXVezSIqoXyIiJWrlxZtF1IUseKK+6nIdxXX3216qMFJXWmcxfk1VCu6zPZBUN/Wpm/bms4PyLiySefLNrPPvts1UeDuq5w5bp164r2kiVLqj4aWs6MY33tyJEjVR8dOy6YrX3ccvSYuzGgY9kFvPW81nPGXVv1Ou7C7hrYz5jMceyuDcr9EEC33W3DkB8BZfaFro+7Ruu5llmXURUydkb1oy7ndB7DvSMifi0inmia5tGTr/12/OQm6ctN03w8InZExC+c3ioCAABMnsE3S23b3hsRr/d/4W4ZulwAAICphAreAAAAHabNRLpDjOoZtXsOqsvOFJPMFIbMPLvN5AmGPLsdVWYpUyhyVM+ohxQ4c5kgLVK4cePGou0yQlpw75JLLqn66MSS7rP1NddHsx4uH3LRRRcVbT2eLh+ifVzxvOnG7T+VyU245ehkyS7v8+CDD/Z+/igmPx1nQdhRfdYoJqE9E2ih0ohhOVyXa8rQsZyZfD2TadVJozPfIZNdyHgo/rIEAADQgZslAACADtwsAQAAdOBmCQAAoMMZFfDWUKILcY4qYNg3i3Pms10fDfS5MJwGA902ZYpk6rI1jJwxzgJjo5KZlVv34be//e2ifd9991XvmTlzZtH+yEc+UvXRWdWPHz9e9Tl48GDv+mkRxUyRQFdgUmlIXQtbvt5nTXfToQiemsjzSj8rU2zQjTf9UcF5551XtDOFLB09p0d1HDIFTYecDytWrKhey5zTb37zm4u2+5GQ7i+3L3S73HL6vsPcsdJjvmzZsqrPvn37etdvOvwQgL8sAQAAdOBmCQAAoAM3SwAAAB2mTWYpU+hQn7GO6rmnKwSm65OZ/DEzWaY+f3bP0IfkkRwtIOYyS337fbLzLEMKYGYyS0ePHi3arvjgvHnzirYr6Ng3kah7nxs7WvzNHV/ND+h4czkFfe3cc8+t+uD0aO4kos7yjCuD4wwpODl0AlfdjkxxxEzhxXFdd8a1XM03RtT77/zzz+9dTiZTlXmfO3Z9k8O7caLfGa4w7/3339+7flMxo6T4yxIAAEAHbpYAAAA6cLMEAADQYdpklvR56tA6RkNqlbjlHD58uGjrs3j3/Fn7ZGpmuLyDLsc9Z9fX3ASHWudDa+5E1NuuGQS3DZnnz5msUUbmffpZrh6MTpSr9WBcjkLfo/vTLcdlFzQf5Y65fr7LtPTVVXJjX19zdZZGJXPMRzUuphI3gbGa6vWaMpN1Z+r7jGo7M3mpIdx5nrme6fv0vNd2RK6mnp7TmQyr2zf6Wibbqdd2d33RiaXXrVtX9bnzzjt712/ImHT7YpzXC/6yBAAA0IGbJQAAgA7cLAEAAHTgZgkAAKDDtAl4q0wgbFQTT7rQX1/I2k06qMFdt36ZwpoavHMBUl22hpEjcqFhnZz12LFjve/JmMzgrtsXSgP8LnStgXgNakfUxSR1/0XUxzhTnO7ll1+uXtOCktp2x0on+tV2xMROcnkmBLrHZVSFK0e1nEwYv+94umurrk8mjDwqQ8e2vk/3sQs1Z4o+6ra7H9Qo91n6fZWZxL1vmyLq6+KqVauqPuO6fkz0tYK/LAEAAHTgZgkAAKADN0sAAAAdpm1mKfPsdugEkcoVFFOZySkzk0hq/ihTuM/lo3Q57rMyEznqRIk6Eey4sgOjlJlIVwuu6XN1lwvT7NOOHTt6+8ydO7fq05c1ioh48cUXi7ZmySL6C5q6onKaJ9i6dWvVZ7pmDLqMszBeZrLTvn2RKQI51KiWM4rjOdUmUHXfK0P2l557bjt1/7nrr66Pfoc4o/re65vsOSLiwIEDRdsV3V20aFHR3rNnT9VnOnyP8JclAACADtwsAQAAdOBmCQAAoAM3SwAAAB2mbcDbhe40gDY0gKgBTReq09cWLlxYtDNhPReY07B2ZgZ6t34axM4URXMF4jSwNx2CeH1cANeF5E/lwtFacPLLX/5y1efRRx8t2jpOIupj5QpO6n53Y1vXx4XA1eLFi4t2pmDnUJkihhNlnON4Ms+RUe3jqbaciTKq8LueexdddFHVR/eF+xGJXv/d+mXGmy7Hfa/oua/XPPfDE90Gt51Lly4t2rt27epe2SmKvywBAAB04GYJAACgAzdLAAAAHZqp8Ay5aZreldBn3y53MpFZgdWrVxdtLbzlnt1qYTKXg9Hnwi7XpBmlV155peqTeZasOSaXV/nmN79ZtI8fP171eSNwmS89Vi5rpDJFDF0RVM1UueyCrqPmEtzxzRRT1fVz2QVd9pmQcXHHStcn0ydTADNzHDImcjmj6JMp4JmZ9HWchuxTPYdvv/32qs/69euLtsuM6rmWKWSs2cWI+vx01wL9HtHvp3POOad6j9LiuRF1lnPbtm29y5lgD7Vte11fJ/6yBAAA0IGbJQAAgA7cLAEAAHTgZgkAAKDDtAl4AwAAjBgBbwAAgNPFzRIAAEAHbpYAAAA6cLMEAADQgZslAACADtwsAQAAdOBmCQAAoAM3SwAAAB3qaY6R4mbCPpUr9qmvLVy4sOqzfPnyor148eKqz9/8zd8U7QMHDlR9dAbrN73pTVUfncl+CDcLtr7m9pWuzyjW5Y0kM9s9po/zzz+/aLsZ6PU1fU9ExPz584v2/v37qz6HDx8u2jrbfEQ9ltx403P4Rz/6UdWnz3Qcx+5aqn784x8XbXcNnD17dtF23wfr1q0r2m9+85urPnrtdNfSzPX20KFD1WunOuecc6rXfvjDHxZtdzx1XBw/frzq8+yzzxZt3X+O26d/93d/1/u+ofjLEgAAQAdulgAAADpwswQAANCBzNJAfc9G3bNbfca6fv36qs+iRYuKtntGfcsttxTtO++8s+qjz4lH9SxXn327fIF+ltuGvswXumVyHToG3Xv0OLhxkllOpg9e38svv1y0zz777KqP5lduu+22qo/mmnbv3l31+epXv9r52c64jufQ5WbG7bhk8jRr1qwp2h/84AerPm95y1uKtrtO6nnl8lKadXrppZeqPjqeLrzwwqqPZpJ0OW4f62snTpyo+mhG7sUXX6z63HXXXUX7u9/9btVHue/YceIbCwAAoAM3SwAAAB24WQIAAOjAzRIAAEAHAt4TSEN1a9eurfpowUkXYvvkJz9ZtP/bf/tvVZ+jR4/2LmcIDTe6UKIGBV0gMlP07o0SEtbQZiZAOqrlZPbxqPqoN/IxV7ovfvCDH1R9Nm7cWLRdwdp3vetdRVt/MBIRMWvWrKKt14qIutigO8+1zxAusJwZtxMZ6M6EyS+44IKifccddxTtiy66qHrPwYMHi/arr75a9dH97o6VFnnMBLzdPp45c2bRnjFjRtHWIprus92+0XFy3nnnVX2WLFlStDPjYqKvFfxlCQAAoAM3SwAAAB24WQIAAOhAZmlM3PPUiy++uGjrc+6IiM2bNxdt9+xWJ9u9+uqrqwvhuJAAACAASURBVD733HNP0R6ag9Fn3ZqlcJNnZibUdEX33qhG9ey9r9Dn0DGguQmXVcnkpShc+foyk9LqxLlHjhyp+mhmRCfWjYi47LLLirbLwWihSjfZrq6PK27ZN3aGjsmJlMlHXX/99UVbM0o7duzoXYbLLOl1MjNBu5tgWa8Nbjl6bdfJdl1uTblJcjUL5Y750qVLe/uoicytRfCXJQAAgE7cLAEAAHTgZgkAAKADN0sAAAAdpk3AezLDoa54ngbvMmG4efPmFW1XPMyFLZXOJP7rv/7rVR8NeA/liuOdKjMDvdt/ur/eyEYVVNRQZKYQaeY8ygT2dRtc2HyiA5kTYVSFNYcEWvfv31/1Wb9+fdF2weIrrriiaD/++ONVn0OHDhXtc889t+qj40KLXUbUxQY/8IEPFO1ly5ZV79Ht+spXvlL12b59e9Ee5/UkE47W0LyG7zPnUKbIp/vBjwao3XePboP7cYD20c/KjPXMZ7t9oYUq3bjYtWtX9dpE4i9LAAAAHbhZAgAA6MDNEgAAQIdpk1kaV0Yp8xz2rLPq3aTPyHU57vmzPvfftm1b7/q47f72t79dtHVi3Yi6cOXOnTurPuOi65zJNVGg8PRppkX38Zw5c6r3vPjii0V76MSm+lkun6TLdsd8uuWaRjVuM9cczTjqRKwREatWrep8T0SdLdKCihH19ctNBKvLfuc731n12bBhQ9FesGBB0XZFDG+++eai/c1vfrPqM5GZRz027hzR7crkBzMFfjNFKfWz3Pmqy3HZor6ilO7c1EKaLquor7mxrevsiqmSWQIAAJjCuFkCAADowM0SAABAh2mTWcoY12SemfoXmeXoBIcus5RZzqZNm4q2e9Z95ZVXFm2XWeqbJDdjaMZlMutknQn5KJc50HGq4+3yyy+v3qM1Wh5++OGqj2ZT3KSqmktwGQ3NPJwJx2FU9Dxyk6GeOHGiaGvezL3PTdateUbXZ+HChUX7wIEDVZ/FixcX7auuuqrq05flcbWiNK/iJuidSHpsXOZG84B6jriJw7XWUSbvk5nE2i0nkxfUc1avMW4b9Jx2Y0mX62oJXnjhhUXbZZaUW58h32FZ/GUJAACgAzdLAAAAHbhZAgAA6MDNEgAAQIczKuCdoeG3TBjZyUwWq2bMmFG0zznnnKrPypUri7YLrOlrTz75ZNVHw5cuEKzrnAlra7jRFSrTz8oE5MdpKgWJ3TjR/e4C+7rfM8v58Ic/XLTf8Y53VO/RkLAbkzrRqgsWZ4oE6jYwmfL/T88jd95nCna+8MILRVsD/BH1pKV6XYqoj41eTyLq45mZPFmvJzoBbUR9vuqkuRHju8a48yozYbC+pvvYFQfds2dP0XbXX3c+9nHXD33NbWffd5r+wCCiPr7uhwn62e681/G+evXqqo8WJx363T0Uf1kCAADowM0SAABAh9O+WWqa5k1N0zzSNM3XT7bnNE1zd9M0m0/+O/v0VxMAAGByjCKz9C8i4umI+PuHtp+JiG+1bfvZpmk+c7L96RF8Tq/MM0x9xqqT20bkshRaNEuLq73lLW+p3qPPYd3zaC1mphMVRtQF0NwzfX1OrM/QIyKOHTtWvdZHl+ues2vmIJNlmI6GFLt0fVzGYEifW2+9tWi///3vL9quAKAWkfvABz5Q9dG8yl/91V9VfXS7XL5BJ2N1Y0eLp7qsx2Rn4EZBMzd6fIfm7ObOnVu0NcMUUZ+P7nqnRSndhLc6dlz2STNTuhw3rvUa6MbAuGT2u/vO0O3UQppurGtxRleIUa//Lsumx9P10e1yffoyhe67SMdxpijlN77xjarPmjVrirbL2qlplVlqmmZZRNwWEb9/yssbIuKLJ//3FyPiQ6fzGQAAAJPpdB/D/e8R8T9HxKl/JljYtu3eiIiT/y5wbwQAAJgOBt8sNU3zgYjY37btQwPff0fTNA82TfPg0HUAAAAYt9PJLL0jIn6+aZr3R8S5ETGraZr/JyL2NU2zuG3bvU3TLI6IOigREW3bfiEivhAR0TTN1CmEAwAAcIrBN0tt2/5WRPxWRETTNDdHxKfatv3Vpmn+l4i4PSI+e/Lfr41gPUdGg24utKZcQUcXlj3Vr/zKr1SvLVq0qGi74n6/+Iu/WLTdbN979+4t2jt27Kj6aJE0V+Tr2WefLdouSKlFxjQo7gKRGuJ0QbxMccupFALPhNQzfVwxOA2MusBt5rNuuummzuW4WbozRe/WrVtXtN2s6zqDutsGLbj6sz/7s1WfP/iDPyja/+k//afe9ZtM7jjoOeHOEQ2p63FwY18/a9myZVUfDWYfPny46qNB7ExRSp0VPqIO87qChH3XW7dcd82bKJlz2AW89fjpcvT8iKjD7e680mun+y7SsHamYK3bTj0WuhwN3kfU3we33HJL1UfD7n/6p39a9Vm6dGnRdvtYX3Pf3UN+dJM1jjpLn42IW5um2RwRt55sAwAATEsjme6kbdvvRsR3T/7vQxFR314CAABMQ1TwBgAA6DBtJtLV57kuB5OZJFef3WYKW2WK4L3zne8s2tddd13V56GH+n84qM/9XS5Bn3+7Z9SzZ5eF02+44YaqzxVXXFG0XYE43V/6jHrz5s3Ve7QQniuMpwXZJlJmEknNKbgMSd97HPcMXbMAroCdchOQ6vHUwpAus6TZNs0XRNTniBaXjKgzGzpJqFsfd+4tX768em2yuP3livmpzMTISseSu+bo+txxxx1VH71+uIyLXhtcAVHdBleUUq+3buzo+ug2uO0c18TX7rzXz8qcw24M6H7WfeM+W7/TXGZJx87QbKdup9vvmm/T7XQZIT3vt23b1rvczHeu21+aqXK5pkxedij+sgQAANCBmyUAAIAO3CwBAAB04GYJAACgw7QJeGvQzYW7NCDnCp4tWbKkt8+8efOKtgvcanBR37Nr167qPRrEc+FtXa4r7qdBXldgTNd57dq1VR8tQukKFOpyNHh3/fXXV+/RMKHbF3/0R39UtLdv3171GZVRFCpz4UtdztDgYiYQrMH6n/u5n+tdH31PJtjuwtuZsOrOnTuL9ty5c6s+WsTQFblz5/Vkcdup+8KdM+6c7ZP5EYkW9fzwhz9c9dFiuW4sZa6TmUKHGtZ2ffSc0Gug23+XXHJJ0V6woJ5etK8osOOuk0MCwC7orMHmK6+8smi7Hzzo+M/8+MP10XB55pi78db3gxV3vupYf/TRR6s+DzzwQNH+xje+UfV55plnirYrpnro0KGiPXPmzKrPuH4cEMFflgAAADpxswQAANCBmyUAAIAO0yazpFmGd73rXVUfLQzpCrLp83CXR9Jn0vv27av66DPfgwcPFm333FjXJzOxqduGTCE1fUbtikBqLsEVntPn2JqDcftP3+OyWXocdu/eXfXJFPPLyDzH7ss1ZdbF7QvNRLh10fetWrWq6nPjjTcWbZdB00yBjh03TvS8ymSqXK5J+xw7dqzqo+PdjW034fNEyRS1zeQ4lNtOvX7oZ7mx9Nu//dtF2+XoNm3aVLRdfkrHoMuJ6fpkJnl1Y7vvWuXGm15jPvrRj1Z9fvd3f7d6rc/QAoV6TXbXUs0sXX311UVbM63utcykzO4c1j5un+r1y/XR7dTz3I0B/Q5x1/G77rqrek1t2bKlt49yxVT1GpMpIpvFX5YAAAA6cLMEAADQgZslAACADtwsAQAAdJg2Ae9LL720aP/zf/7Pqz5apMoVOtRgmxZmdH0cDbutWLGiaOvM3u49LuCtYVBXAC0T3M2EmnV93Hs0dKjr7EJ/mdCrHs/777+/6jOqgHeGBmp1X7jt1GCsC8jr+9773vdWfTQMun79+qqPBnUzoWH9bPcePZ5uLGnw0wVRtZiqG/96XrnArRaNzYTm+z4nIrcNWvTRXRs0uO7CvrrszKzw6ld+5Veq19797ncXbS32F5ELb7ttVzpW3D7PFFNVM2bMKNquKKXur9/4jd+o+nz+858v2u4HBUO4fZMpMnrvvfcWbR1Lbjtffvnl3j6ZgrC63936Zsa/ro8eX3cN1IKm+kOniPq6mDk/3fdeRqa461D8ZQkAAKADN0sAAAAduFkCAADoMG0yS1ogSzMSEfXzSpcJ0ue5bjI+LYrmMjj63FXb7pmr5glcRkjf53IAmefYmefs+gw6U3hO+2QmG81kZYY+o9b1cfsrU6hM11HHknsWrq+5CT9vv/32ou2KqWYK9elzf7ed+j7dpsyxchkh7ePGrR4Hl5XRXILLBOk+XLhwYdXHTUp6KrcNeqx0otOIeqJft7+OHDlStF3ORMdyJgOp6/M7v/M7VZ8XX3yxaLv9p9lJlzXKZKp0H7pzWJfjzuG+fJT7bM296MThERFvfetbi/b3vve9zs8Zt7/+678u2k899VTR/tmf/dnqPZrbdN8zeu3SXFFErjioct8Pesx1uW4M6HXIZceGTGQ+JKs4bvxlCQAAoAM3SwAAAB24WQIAAOjAzRIAAECHaRPw1lmJ9+7dW/VZvHhx0XaBQw10uwCiBu1cCFz1FQSMqMNwmSKQmeKDQ4pJuvVx+0LDeRq8cwFNXa4Lwep2ue3MBPp0febPn1/10R8DuHGh66jbtWrVquo9GjJ1xSTnzp3b+9mZHyboDxxcaFjXWZfj3tMXCo+ow9pu/dxrSo+VW59Zs2YV7UWLFlV9+gLe7ny98cYbi/a6deuqPnoePf7441WfrVu3Fm23DXruux8H6H7+l//yXxZtF8DVGd3dDwp0JnsXAs+sn54P7kcRmfNTrwWZMaBj6dChQ1WfNWvWFO2hAe++61tEfV5pYc2Iej8fPny4aD/22GPVe97ylrcU7RMnTlR9dL9nfrjj9mmmyK4ec91Od+3S/eW+lzOBbl22+4GIjrfMdo4Sf1kCAADowM0SAABAB26WAAAAOkybzJI+z33yySerPprrcAW89Fno0GfAfUUp3bNTzSm458b6fNzliDJZAf18ly3qKzgZUWcOMs+N+/ZNRP2MemjBsaVLlxZtV2xQs2xuX2g2RoshupyCTiLptlPHoDuemhVwz+v1OGSKb2aOg66POw46ltxnZ8aFji9XhE/Pc818Zbj8lB5PLUAZUedMXJZHl+2uH/o+l6HasGFD0dZx/P3vf7/3s1evXl310aKAmUKz7pgPyUW6caH7J1MIVwt/ajHOCJ+DHEK3yx1zzeW4Y67XB80w7du3r3qPXodcZsntH6XnWqZgrTue+j5dH7d++p32wgsvdK9s+GugvpYpHOwwkS4AAMAk4WYJAACgAzdLAAAAHaZNZknzDZs3b676vPe97y3amefhmhWIqJ8lZ56VqkzWyOU6MnVIlHu+q7kX94x6yGSZmclZNVvh8iHbtm0r2kOfNWtO7R/+w39Y9dHci8to6HboNrj10zySm9xZcwAu+6R9XJYnc6wyY0Xpclweo28y1IjceFPuOOhY1rpBGZoli6jzPXPmzKn66LFxNXfWrl1btF0uUrd9yZIlVR/N0WktoSeeeKJ6jy5H63xF1Nc3NyYy+Te9BrqcX997IupzIpOp0n3q9vHRo0d712dUhtSTUu6c1vNKJ3CPqM+rTJ22TM4p0yfzPajr7M57zZcNzTxm+owTf1kCAADowM0SAABAB26WAAAAOnCzBAAA0GHaBLw1bOYC3hqicwFNDba5QFom/NY3oWwmLJ0pAJgJ17rP0iBgpuBk5rN0O124XO3YsaN6TQPeQ2mxt3vuuafqc9VVVxVtFxrW1zQU7gLBur/cWNKguAvB6j50RSkzwXodTxq+dMdXl+vCtDrWMxNqukKCmQlS9bx+7rnnqj59XIhej5/ro4Urly1bVvXR/XXppZf2fr7bX0899VTR1gl677vvvuo9N998c9F2k+TqMc4UInXXBh2DbtzqeHfjXydGzgTH9UcGrqDjQw89VL3Wx43/THB9FDLFQd21VM+rzA9h3HHQceC2u+87zX1fZX6QpMfTHYfMWFIEvAEAAKYQbpYAAAA6cLMEAADQYdpklpTLO2zZsqVouyJfml1wk27qc9jMM2B9xu+e7+qzWvfsNvNsOVMkU9c583zXZbz6sgtuH+vz+U2bNlV9nn/++d71ydDJT7/73e9WfR5++OGi7da5LxPkjpUWCdR8RkRdNNMVTNQx6JYze/bsou3G14EDB4q27htXyE/PI5c10iyFFj6MqPMzmUk3XT4qM9lun1WrVlWv6UTJLkekhSLdeaavufPz2WefLdpaiDEiYufOnUVbj5WjeRU93hH1uNDxF5G7Dunxc9dAPTZuffR4Hjx4sGi7DI4uxxV71eVkZPJJ48o1ZbJjQ5eTmTBbj4O7fvT1yewHV4BYZZbjtiHznZbJOg3FX5YAAAA6cLMEAADQgZslAACADtwsAQAAdJg2AW8N5bpCZXfeeWfR/pmf+Zmqj84+npkNesgM6plAtVuuLicTAnfLyayzhlVd2F2DsLo+bgZpXT9XQDRTXC1Dw8cupK7hY1f8TUOHmfXTQoJu/+lnuXGhfVygVbehb5Zzx312pqjckCKtjm5D5ocKrk/ftmsYPqIOzbs+evxcCFy3wR1zHUtu3+h5pD8WuPbaa6v3XHbZZUV7/vz5VR+d4d1dP3Rsu8C+boMLqeu1012TddluOUr3u/vBg27nqGTCx+54usD0qdw1J1NAVAs6ZgLemWu/O4f61sdto+4vV+x1yPpl+kxUQdG/x1+WAAAAOnCzBAAA0IGbJQAAgA7TJrOUKU6nk266SS71uazLRAx5FpopZJmZLDBTDDFTnEtzJpmiaK7PzJkzq9dO5faV5gm0AF9EXexSn82/3rKVPnt3z9U1L5CZVDiT5cnknDLboGPQLUf3lxsXut91n7rt1s9yy9XXMtuZmRTUZT80r9KXBXHcJLQrVqwo2jq5ckS9/3Qy5Yg6l+O2M7MNy5cvL9paPHLt2rXVe3Qfu4xQZtLSBQsWFG03cbO+5rIoes4sXbq06qPrrIVIXXZGP9vlnDLFD0dFx+mQMenOPd037hqjmbjMBNqZ/GDme0+X43Jiuj4uRzekUCRFKQEAAKYZbpYAAAA6cLMEAADQgZslAACADtMm4J2hQcD9+/dXfTTYpkXmInIBOQ3DDQldu3BjJvSXoSFcF+jLzD7eFwB279FQogtEDplNPiMT8MsUnBxS9DET/HRjS0O5mW1w40LHf2a8ZYwi4Oo+34Wj1ZDApiuC+tnPfrZoZ4LsLtCq275o0aKqz4UXXli0NVAd0R/ofuyxx6r37N69u2gfPny46qPnlfuBgR4HN5Y03O72he4vNy502Vpw1V2XDh06VLTdNdod4z7umGd+gDF0vJ9q6DmtMuewW1/9fLfdfZ/vrpv6WZkCthlD9xcBbwAAgEnCzRIAAEAHbpYAAAA6TJvMUiZ/oa9pkcqI+nm4m7S0r0ChWx99npspSpkpOJkpoOie3WoWwG2nPm92uQR9n35W5hl6X2HLN5JR5B9ej47TzLHJFFPNZCl0nGaW49Yvkxfs24eZa0MmU+gm0tXzyuUiNWOjEy5H1Jml7373u0U7U9TT7Yfjx49XrynNNbljpQUvxzluJ8pETryq4yuT5XF99JqcGRfueOpE9FocNKI+xrq/3HeIrt+oMktTEX9ZAgAA6MDNEgAAQAdulgAAADpwswQAANBh2qSxMmFVDbbt2LGj6rNnz56ifckll1R9NKDpgm0aZNMwnAtEaqA7E5x1gT5djgv06fq55WSKR+pnZcLvmVnqpxo9FkNmuB5a9E65farHxh3zISHczHsy2zCk4KXbX6MIEmeOg9t/Q4qVuuKqWizSfdaBAwd6P2syTfVAd9/5Oh3oNTnzfTBjxozqNT1W7ocJyn2n6fpoO3N+ZD47I3O9ncjAfgR/WQIAAOjEzRIAAEAHbpYAAAA6TJvM0hDumeaWLVuK9vLly6s++mzU5X2GTKQ7JHeluSK3bLdczSy5gpPnnntu7/pofkbXx+VrdP1mz57d+zmTrS/zkMlEZJ6hZ57FT/W8yKhkJvMckkUZml/JfLb2ccdTz4mJzOxlChSqic5+jMJUzyjp+rnrpF63M1k7N5b0+8l9H+j7Mt8ryhWcvOCCC4p2ZmJ6d33TPlNxTPKXJQAAgA7cLAEAAHTgZgkAAKADN0sAAAAdzqiAt84I7sJwWqjShZy18FcmQJ0JHGqAzgXdtI8rQjakuKULeGsBMbc+R44c6VzuiRMnqtcOHjxYtB988MHe9csUYjwTZIKeGS5sOarCkGoiw5ajCO4OXcaQc3hc+8YFcDXIO7SoZ+Z6MZWMqtjrUEOC//pd5K6/eoxdn1mzZhVtt91aGNX9IEmLULrvlb4wuVs/9fLLL1evDSl8OxUD/PxlCQAAoAM3SwAAAB24WQIAAOjQTIVng03TTNhK6DPgX//1X6/6XHrppUX7wgsvrPqcd955nZ/jsinuNaW5EzcxoU7CuX379qrPE088UbTdpMKHDh0q2u5Zt67PkMKajj7Td7mE1157rXc5E2VUuYlMUcrpKJODGZL9mIrF6XDmmMwJedetW1e09bspImLhwoVF22WNZs6cWbTd94z2cfR6u3v37qK9b9++6j06Mf3mzZt7P2cKeqht2+v6OvGXJQAAgA7cLAEAAHTgZgkAAKDDad0sNU1zUdM0X2maZlPTNE83TfP2pmnmNE1zd9M0m0/+O/VnUQUAAHgdpxXwbprmixFxT9u2v980zdkRcX5E/HZEvNi27WebpvlMRMxu2/bTPcuZ/glXAAAw3aQC3oNvlpqmmRURj0XEJe0pC2ma5pmIuLlt271N0yyOiO+2bXtZz7K4WQIAABNt7L+GuyQiDkTE/9U0zSNN0/x+0zQzImJh27Z7IyJO/rvAvblpmjuapnmwaZr+uTAAAAAmyencLJ0VEW+LiM+1bXtNRLwUEZ/Jvrlt2y+0bXtd5o4OAABgspzOzdKuiNjVtu3fnmx/JX5y87Tv5OO3OPnv/tNbRQAAgMkz+GapbdsXImJn0zR/n0e6JSKeioi7IuL2k6/dHhFfO601BAAAmERnneb7PxkRXzr5S7itEfGx+MkN2Jebpvl4ROyIiF84zc8AAACYNG+4ueEAAABOSv0a7nT/svSGNVETMGYmKHWTvGb8+Mc/LtpuAkadyFQ/y010etZZ5bD64Q9/OGj9MjL7J3NsdJ0zEwYPsXr16uo1nZT58OHDVR+d9Picc87pXc7s2f31YI8ePVq03WTKmT4ZmbEz1ek2uHNPx875559f9dHXdIJUN/70+LpxrRNUz5kzp+qjn+XG28GDB4v2tm3bqj5qyCTRZ+rE0uOixzdifNfXzHecWx+l6+fOGf0s1yfzfTXO7xqmOwEAAOjAzRIAAEAHbpYAAAA6cLMEAADQgYD3FOdCbPqaC7UNCc9qgM7RkJ8L/Y0zZKf08zWoHVGHZd0+HRLonjVrVtG+9dZbqz7vfve7e9dPw9quj4Zwzz333KrPihUrirZu5/79dX1Y/exXXnml6nPPPfcU7T/5kz+p+hw4cKB6TfX9WMD1mUiZQGtm/TS8ffPNN1d99FjpZ7lA9fz584v2jTfeWPVZuXJl0X7ttdeqPvqaO+b6+b/7u79b9dGg/5Bg9lQLc09k4Pzss88u2u76q6+5a+uQH05kzr3Mdo/rWp/5LiLgDQAAMIVwswQAANCBmyUAAIAOZJYS3LPRTHZHDXm27LI04yqYmKHr7J7x62vjzKHoZ2X2TeZ5uFq7dm312qc+9amiPXfu3KrPsWPHirYWl4yI2Lt3b9HWLENEXZDw5Zdfrvpo8UjNPl100UXVe3R/XXjhhVWfj3zkI0X7F3/xF6s+jzzySNH+3Oc+V/XZunVr0R7XuBiaO9HzMzNO3DZccsklRfvtb3971Uf3u+Z/Hn300d71mzlzZtVHr1U6/iLqMeiKjOo4WLhwYdVn586d1Wvw3Dmtr7l8mY5BV4xWx5LLPGbG8pBCy/qeTB7ULVe3y13HNY800flG/rIEAADQgZslAACADtwsAQAAdOBmCQAAoAMB74QhgWAnE0jTcN6owtyjmq1ag8ZDC2KOajuHFIxzIUQ9xhpw/cQnPtG7nM2bN1d9dDtd+FiLDbqg+PHjx3uXowFgXT/3Hi1I+NRTT/X2OXHiRNXnPe95T9H+uZ/7uarPf/gP/6Fou8Kauo6uYGKfoUUEM+e5Hk93Xl1wwQVF222DniMa1r744our9+i4WLBgQdVHt90dc10/92MB7aNFNCPOzIB35scBLqztQvJ9/13PV1coUtfHLUfHoLuW6rh1123drr5tisida3odyuw/dxwyhoTUs/jLEgAAQAdulgAAADpwswQAANCBzNIkcs+otTjX2972tqqPZiC2bNlS9Xn11VeLtnt2q5mHZcuWVX102a7I3RCjymINKfTpsimaxdL9rpPmRtTHweUA9H2uMKR+tvusJUuWFO3MBKnq4MGDnf89wmeWNm7cWLR37NhR9dHxtnz58qrP4sWLi7ab2FePzWROtpspRuvWRffFrl27qj66fzQjtHr16uo9M2bMKNpu3+h5lcm2uSKGul3z5s2r+pyJMmNrSNZzaGFjfZ+7bmrex40LXWdX3HJIZknHl8sh6nXJ7b9MtlNfc9fxcU7MzF+WAAAAOnCzBAAA0IGbJQAAgA7cLAEAAHQg4D3QKIpfufe89NJLRfsDH/hA1WfRokVF+y//8i+rPs8880zRdoHbdevWFW0NmUbUM8U//vjjRfvIkSPVezTQ57ZTQ4guBJsJgWsgM1NUTgtORtT74td+7deKtiuktnfv3qK9Z8+eqo+Gt2fPnl310VCkC5m++OKLRdvtG92nGuJ0Id0XXnihaD/55JNVn02bNhVtV2TxvvvuK9rnn39+1UeD67r/nImeWfxUmSKVbmzrjyKWLl1a9dEfEOixccUutXClWz99TUPhbtkucKvjy10b3qjcNcaF5E/lxrGe93qtiKgL0XU1lQAAIABJREFUhrof2GSKz+rnu2PeF+jOhNTdj0z0s/r21evR/eW2YVQFpB3+sgQAANCBmyUAAIAO3CwBAAB0ILNk6DPfzDPgITI5J5ev0ffdeOONVZ/rrruuaLtCh5qT0PyKW84111xTtN1zbJ301eUv9Nn75z//+aqPywCpIRPyanHEiHoiWD3mmhmKiDh69GjRPnDgQNVHn6u746n7wo0LXbbLDWkxxPXr1xdtVzBOC0y6HJG+zx3PTCbiqquuKtqaq3Mmsyhlhrs2aFbM5Sg006X7dM6cOb2f5fZD5pqin605SbccV0z1jUKvcW4f63jPfIdoRkmzqBH1+eiO1ZCcjssN9WWW3HjTcZspZOmu0ZoJzeRcXWFNvQaOEn9ZAgAA6MDNEgAAQAdulgAAADpwswQAANCBgLeRmQ16iEwhSy1a6AqVacjPLUcDfC4EePjw4c71i6gDexrEc4FDfY8W04uoi9wNLSaWOTb6We9+97urPhq83rlzZ+9yt2/fXrRdwFs/2wUgdR+6MPnBgweLtgt4L1iwoHO5J06cqN6jRUUvv/zyqs+hQ4eKtgtoajjVBUhvuOGGor1t27aqjxbAdONinCHOn5Ybf3psXLBew6q6TzMz0LsChfo+9wMMLVTpCq7qPnZhWt2GqRS8HyXdp2479TqtY8Adh4svvrhou1D/7t27i7Y7H3TZro8eq0wQW7kfdgxZjqPFLF0BXd0/+v0VUQfttajn6eAvSwAAAB24WQIAAOjAzRIAAECHMzqz5ArajSuPlKHr454t63NZNzGhPid2z5L1ObbLkGgfl0vQz9J1ds/iteidy8roM/5MMUlHP0sLYkbUObBVq1ZVfXT/aMFJV2RRc0RuvGkexD1DzzxX1/VzE5vqPszsU80GXHHFFVUfPX4up6aTxboCe2vXri3a73rXu6o++/fvL9puguWJkpmU2Z1X+r7MBKSaHXMT4Oo4cdktfV9m/TI5OrcvRiGzjydbJovlMoSncueijgtXOFi/D9znuOur0m1w40KvyXotdcdlyITy7rP1Ou5yfloY1X1/uondR4W/LAEAAHTgZgkAAKADN0sAAAAduFkCAADocEYHvJ0h4UEX0BxSRHFIwNuFfzUsmAl4OxpadkFGDZjrZ7ngogYO3bpon6EF7ebPn1+0XVHFHTt2FG0XmtdgoBb8c4X7XKBb6Xa5ALp+lgvu6j50xUr1s3TsuAC/bpfuz4iIJUuWFG0X0NTAuQY23fosW7as6qPvm8ywb+az3TmsgWm33/W80fdkCsS6c0bf58a6vuY+Swv+ZQpgDjHZYW49h93YHnI8dVy4a7See6tXr6766P5xP8DQ67i7Ji9cuLBou0LBGur/xje+UbRd0d25c+cWbS1g67jvAw10ux/hXHbZZUV77969VR8t4jlK/GUJAACgAzdLAAAAHbhZAgAA6DBtMktDil9lnukPzSX0cXkWfbbsCm+tWbOmaGe2IfP57j36zNxlZfR9ulyXichkobTo49AJGTXjosURIyKee+653vXRzI32cUUpNYfgxonmB9wkufpZLnOgGSWXrdDP1wxCpjCpG5Na1NMdc81AuPGvWQ+X/XD7ZyrLTGzq8lsqMwGucue0ro/bx/o+tw3jnJB0Ksmce3reuP2uxysz+a4WWXTnnk62u2LFiqqP5iDdea7FSt0k1jqZ+LXXXlu0//zP/7x6T2Zc6L5x+S3NVLn8livEq8ZVPDWCvywBAAB04mYJAACgAzdLAAAAHaZNZmlU9Th0OS5boc+Xh9RZcuurz5LdZKMLFiwo2q6+SSYToevntlOfN2cmXtVsUWZyRbevdBLaoRPpaiZD919Evb/c+uix0ZyOqz+kk1q6rIC+T+uSREQ8//zzRVuzDBF1nsBlIHQ7NUfkajPpNrg+uo9dH10/l3/TseMmi9X3Zc7PqU73TUS9TzOT77r8kdIcosva6Xnvrh/6PpcXydSNm0oy1/FMHsmNSR3bmTyoXv/dONm3b1/R3rhxY9Xn0Ucf7XyPWx933X7b295WtHXi6/vuu696j2ZPHd1/Wi/JfXbmO83tU5eHGhX+sgQAANCBmyUAAIAO3CwBAAB04GYJAACgw7QJeI+KBvgyYdEhgVIXoNPgrhb9iqgLk7ngs4bY3Gdp+M0VEszQoKfuPxcgVa7YWmbyxww9Nq64pa6zK6Q2b968oq3r7CaevPHGG4u2C+DOmjWraLtjpct2403XOVPkLkPDqu54akDThWB1fVwIVt+n+9x91nQLc0fU2+kmYdawqo4LF1TVse0Cy3oeuRCshmfduafhezduMz+c6DORAf7M+mV+/OGuH3ru63nkiuUuXry4aD/88MNVn29961tF24W39fi565COJ3fu9a3zO9/5zuo9Tz/9dNHWieDda8uXL6/66A9z9uzZU/XR7zA3tsc5MTN/WQIAAOjAzRIAAEAHbpYAAAA6vOEyS0OeabrMjT4vzSxXi5C5Z7daJNBlFzIZnEwGIpPf6psQ0r1Hn/u7HIAuNzMxrKN9XNFHzQa4yVq1KJpul8ss6Ta4MaA5AFeIUZftMiR6rFwGTYtFah/32fpZbrm6L9z5kMnp6HJc4TnNeB05cqTqM5VkMjduO/W1zETSOtbdOZ2Z3LmvgGJEXZTVFSLtm2Q7oj9/NJGZtEyOzu2vzHVIzxstMHnFFVdU79m+fXvR/ou/+Iuqz6FDh4q22196/Nz1Q79X3DHfuXNn0f6Zn/mZor1s2bLqPboN7jp56623Fm2Xi3z22WeLtk4CHlFvu7tWjbMwKn9ZAgAA6MDNEgAAQAdulgAAADpwswQAANBh2gS8NYg3zuJTyhW/6lsfN0u3zuLsQp2ZcOiQALWTKSqn6zNkv7tgpQa6hx5PDWtv2rSp6qPHzx1P3RfadqFEDdO65eoxdvt4//79Rdsdc12OKzzXF250oUkN7rqgvYZMNSwaUQdaXfg4M27HOWv4OLjArW5Dpiir7gt3HDI/KND1yRR7zQTFXbFBDeMfOHCg97MmkyummgkE67Ugs7/0xzuXXXZZ9R4NNe/du7fqo+vsCs8ePny4aGd+UOB+7NFXuNiNN90G90MAHSdbtmyp+mgRSreco0ePFm13DcwUSR6KvywBAAB04GYJAACgAzdLAAAAHc6ozNIoJnbMfPbrff6prr/++uo1LZjonhHrs+VMocPMvnAyBSY1K6P7wj2/17yFy29p8TKXa8rYtWtX0T548GDVR/e7m6RRix/q+rkxoMVAjx07VvXRiS9drkMzLa74mx5P97xeswuaNXLv0X2j+amIiGeeeaZouzzN6tWri7YrTqe5Fz12EREnTpyoXpsoOgaHFkzUnJAb/3reZCY0Vplzxl0bdP3ccvR9LuOyYsWKoj2uzNKoJtt13weZ7xW9TrvjqYVv3/72txdtd8488cQTr7+yJ+k565ajMtf+zHVIrwXufNXvBzdhsJ73u3fv7l0fl5HTfKoWeY6oi6m6HNhQ/GUJAACgAzdLAAAAHbhZAgAA6MDNEgAAQIdpE/DWwKgr6KhFAYcGAzXo5j5LA5padEwDr279XFgvM7u89nHhYw1xuiC2hhldMFD3l+5T99m6/zKzmrvlDOEKQ2ow0H2WBry18KIbA7r/XGhy48aNRXvr1q1Vn/nz5xft48ePV3008OgKHb7wwgtFWwPeLnyp26X7ISLiqaeeKtpaHC6iLjTnQuqrVq0q2i5wO86icn2GBrqVblemyKiOSXcuap9MwcnMj1Pc2M78iEQLkY7LqI6LM6QYrjuet9xyS9HWH064z8mEtTMFdTUE7q4Neu659Xnb297W+Z5t27ZV71m5cmXRXrNmTdVHuaC4HmP3vafb5YL2rpjlqPCXJQAAgA7cLAEAAHTgZgkAAKDDtMks6TN0fZ4aUWdT3LNuLdznCvXpc0/3HFSf1WpRMvfZ+ozaLVfzPS5r1DcJZ0SdVXDPxzOZB32fZinccvV5s8ss6XEYZ1YlMzmx5i+0IJvLCmihvsxEtZmJdF1BR32fFsSMqAtg6na7HIzmADTnFFFnoVwORvNabn/p/tGslntNz+lxGtVk3Zrrc+Nfrzv6We74Krd+ug2Z3KY7h3VcuGvVwoULe9dxFIYUBXa0iGZExOWXX160XVFbPX7r16+v+mhmVa8NbgJcLc74X//rf6366LFxRVuvvPLKov3hD3+46qNZokxeUCcDdhmhBQsWFO2rrrqq6qPHz+UZr7766uo1pVnOiy++uOrjclWjwl+WAAAAOnCzBAAA0IGbJQAAgA6ndbPUNM3/1DTNk03TbGya5j82TXNu0zRzmqa5u2mazSf/nd2/JAAAgKlpcMC7aZqlEfE/RMQVbdu+0jTNlyPilyLiioj4Vtu2n22a5jMR8ZmI+PRI1vYU11xzTfWaFvxzBfY0XObCtBqqdiFwLaqogUNXGEyX60KKGrJzoVx9zfXRgK37LA0uulC6hsA1rOeC2RrkdTM/ayA4EzZ3dNtdcFED55deemnVR4vIPf7440VbA/0RdajfFffT7Zo1a1bVJ3Ostm/fXrTdMdfwp4Y6XbBSj7kbt7pvXNFMPZ5uXGhAVMOhEX7/TDcaynXbpD9wyNCwthsn+poLeOv6uT6ZH1y4H5+Mw9CgvdJxHBGxYcOGop35AYYLtus+1OW4HyR95CMf6V0/vQ65cXPjjTcWbVfQ8d577+1sR0Q888wzRVvPV3cNfOihh4q27s+IetvdDwrmzZtXtF0BXb2+uR/q6DV5lE73MdxZEXFe0zRnRcT5EbEnIjZExBdP/vcvRsSHTvMzAAAAJs3gm6W2bXdHxP8aETsiYm9EHG3b9psRsbBt270n++yNiPr/PkZE0zR3NE3zYNM0Dw5dBwAAgHEbfLN0Mou0ISJWRcSSiJjRNM2vZt/ftu0X2ra9rm3b64auAwAAwLidTlHK90TE823bHoiIaJrmv0TEfxcR+5qmWdy27d6maRZHxP6uhWRpzkSLgEXUE3y6Z92aP3LPPfV5s8uHaBZFi5C5HIAux2WENAfgCtplJsnViWDdcvom83TL1n2qn+Nec1kezXwNnfQ400f3+5//+Z9XfbRAnGZwrr/++uo9Ormty/JoJk4LR7r3uWf6mgNzObq+SS11n0fUuSH32drHFUzULIXLfmhRQJeh0nH6wAMPVH3GZUhRyswE0C4H1pctchk+PTZu/XQ57tqVySoqt5yJmkjXfbYbX30effTR6rXf+q3fKtou83LttdcWbbfdmjfSc9HlfTT75CZf10msM5m+e+65p3pNJ/R2+0+LPGohTZfvveGGG4r2unXrqj763ZjJz27atKnqowVq3TVQs52jdDqZpR0RcVPTNOc3P7li3BIRT0fEXRFx+8k+t0fE105vFQEAACbP4L8stW37t03TfCUiHo6IH0XEIxHxhYiYGRFfbprm4/GTG6pfGMWKAgAATIbTmhuubdt/HRH/Wl5+LX7yVyYAAIBpjwreAAAAHU7rL0sTScONx44dq/poSMzNMq0FAF24UUPfLhytYTcNrbnlaqjOhUO1j5u9XZftipBpHzebvAbkXFC8b7s0vOdec6FhLTDmZuXOzDifCeHqPt28eXPVRwuaasDWFVvT0KZbFw1Vu4KYO3fuLNpuf2kA3fXRZa9du7b3PfPnzy/arjCejhMNv0fUoW83Q7meV1rsL6IOmbowrf6QY1SGFD9078kU+tRzVkPMmR88uB9O6LXKHXP9LLcNes649XEB23EYEubO0h9XuO8VLdb41a9+teqj1ws9j/THIBERL7zwQtF2x0qPjZ6vEXXxZbe/dLtcYU09h/UanSm664pm6nLdD0T0s1wfLZR63nnnVX3cd+Go8JclAACADtwsAQAAdOBmCQAAoMO0ySxpMa6rr7666nPTTTcV7YMHD1Z99LmnexavRRVdsUF9Vpt5bqxZAZdl0PVxfXQ5rliYe+bb91kum6XPzLWPKxinmSVX+FM/e9GiRVWfTGZJuRxYpthg30S+Lp+hx8btC91Ol6PTrJjLgenzebcN111XFsPXbXJ5DD02mW1wOTodF+6c0YyS2wYtVOmK+T322GPVa6MwqglbXfFPpfkUHUsuH6KvufNVl+vG7ZCcn3uPy6X10bGUKYg5KqM6vi4Xo9cYHesunzeEy54ql33Sa8GePXt6+7jvzz5u0vRR7Xcd75k87yjxlyUAAIAO3CwBAAB04GYJAACgAzdLAAAAHaZNwFtDnV/5yleqPhpwdTNIz5s3r/ezNLjogrE6C7yGkV34UoO7LlyeCUdriM19lgYKXQhcC/65ULoGATOBYA2rum3Q1zLHJSMzC7wLHOp26L5w+1j7uKKeGsp1wWcN4br9pePCHU8t7KaFNp999tne5bqgvYbLn3766aqPzpjujqcGMt3Y0e1yYfdxGVX4WAusunGhy9bPduNY91dm/TLLcTI/XnDnRJ+JDHSrTKHPDLcv+goOu/fouNAfFg3V92OViNx2910TI+owuftst9/71seF1PVamjmvRom/LAEAAHTgZgkAAKADN0sAAAAdpk1mSSfP/LM/+7Oqj3tN6bNQ99xTJ0F0hQQ1o6E5JzcBqGZTXFEtLTqmE6hGRDz11FO9y1FuG2677bairZOYOvpZrniePnvXfJfrs2nTpt7PdjQbkHlm7Z6h903W6bZh3759RdsVZNP94/poodQFCxZUffR9Lt+gY+fRRx8t2m4CYZ3g001orOvnsoC6Pt/85jerPvfdd1/RdoX6NE82UZO1us92MhkNPX7u/NRxq213XdLXXK4jkzXSdXb7WD/LTV7s1nEcMhmhjFEVR8x8tn6WGydD1mdoJjNzncyMbTWqfNSQ92Q+e5T4yxIAAEAHbpYAAAA6cLMEAADQYdpklobUQHHPd/V9Lvuxa9euznZExPe///3ez59Muu0uK6Pb5WoAuRpTPy2XEcpMipihz+cztVSGPEN3Y0nX2S1X61S5ySn12XtmOW4yT80fXXvttUVbc3YRdT0kt//mz5/f+TkRdTbr7rvvrvocPny4aGeyH0OyKUPpWHLHPJOT0P2TqXWk25mpheT2jY4dlyvK1L3JjEk3yXefzKTWalRjYFSZpSFGlbvKfKc5mW0fRQZoVLWshn7WOK8X/GUJAACgAzdLAAAAHbhZAgAA6MDNEgAAQIdpE/AeEhJzobbJDPmpTOhvaDAws51axHBIyNqF7FyxPKXbNapg4NDgvwbONUDtioy+/PLLRVuLN0bUk9C646LvcwUAdZ+6Y6X78KqrrirarrCmTlTrJq7V7VyyZEnVR38o8cgjj1R9MmF8DQ0PDf4PMaqJdHUsZSabzvzgIbP/dGy7MalcYdlMEDtTMFS3KxMinkrX6KF0/2Wu2e66pPvCjclRjdtRcJ89JNSf2YaJ/PFHBH9ZAgAA6MTNEgAAQAdulgAAADpMm8zSmUCfLU92Ub5R5EHcs2WXgVC6XRP5nN1lqjSjpFkjze245biinlpM8siRI1UfnVD2xIkTVR/9fJ24OaKeKHfZsmVF+4Ybbqjeo1ksl2vSjIGbAHfLli1FO5OVcfQ4TGR+ZVRjUDMaOpYiIhYuXFi0zz///KLtzk3dp64wqWaEXA5Gzz13rPom+o2ImDVrVtF22+mK/r4RDBm3Q8f6ZGaUMs6EDFoEf1kCAADoxM0SAABAB26WAAAAOnCzBAAA0IGA90Aa7tXQ5NDZoUelb1bziDpw7mYo1yJy+p5MkNfti0whtVEFA4cU6tNg6uc///mqjwZajx07VvXR11xRPj02brt1fVzhyueff75of+c73ynarrCmvubGwOLFizs/JyLi8OHD1WtKixi6gLL2ccH6yZQJPn/961/v7aP7VAt9urC07htX7FL7uM/WMeh+kKFj0J0zW7duLdqZIplDfuTyRpYZb3qs3PVjSGHIIYYWUVajKlI8SvxlCQAAoAM3SwAAAB24WQIAAOjQTIWCUU3TTP5KAACAN5qH2ra9rq8Tf1kCAADowM0SAABAB26WAAAAOnCzBAAA0IGbJQAAgA7cLAEAAHTgZgkAAKADN0sAAAAdmEh3ErlJEVWmaGhmOZmJfd3khX3vcRMnZibL1Ak/daLYccpMRpmZwFJfc5PQZibx1YlN3XL0feeee27vcmbMmFG0L7300uo9ehz2799f9Tlw4EDRdhMG6zF343YqFMCdDO4cOf/884v27/zO7/Qu58SJE0X7yJEjVZ/MBNU6Tp544omqz/e+972i7bZBMSnu6H34wx8u2rNnz676bNy4sWjv3r276qPjYMGCBVWfm266qWjPnz+/aK9evbp6z/3331+0//AP/7DqM9Umwx6KvywBAAB04GYJAACgAzdLAAAAHbhZAgAA6EDAeyAN7g4Jr44q8OqWkwl9KxcG7VuOC3Vmgp4TGehWmf2ufVygWoPY8+bNq/ocP368aLtwtNvvKjPe5syZU7RvuOGGov2DH/ygeo8eBxfi1KD4448/3r2yEXHWWfWlRYPFb5TAtzsfdD9v2LChaD/55JPVe3S8vfTSS1UfHW/nnXde1UdDwldffXXV54EHHija7nzNhL7PBEOu9Rq0d+eDno/uGvOJT3yi8z0REdddd13RduPiscceK9ozZ86s+qxdu7ZoX3nllUV7+fLl1XsuuOCCov17v/d7VZ8M3cfux0aT/QMC/rIEAADQgZslAACADtwsAQAAdCCzlJAp1thX1NAtx2VVxpVjOvvss6s+7vn3T7s+roCibudrr73W+zkTKVOUUos1uvcsW7asaF9zzTVVn4ceeqh3OZoHefXVV6s+mg1wmYM1a9YUbT3mzz33XPUeLXap2x0RsXDhwqJ90UUXVX20QOKQzNyZIpNx0fzHoUOHirYrOKnXC7dczREdPny46qOZFpdHmjVrVtF214ohGZJRZD0n2pB11GOVySVmzvujR49WfTSXpscuor4mu9zhli1bivbSpUuLtjvvXQZT6Zh0+S39jpjsfJLDX5YAAAA6cLMEAADQgZslAACADtwsAQAAdCDgnTCkWKMLBWpobWgI1oWqlQZ1dZbzCD/DvNJwnq6zBoQj6vCxFiN0y3H7OBOKHCIT2NTA4dy5c6s+um8OHjxY9dEwo/uxgAtVq8yPDDSEq+FLV6BQj5ULmeqs4Vq8LiLimWeeKdouiPpGCX1nxte6deuKtp4jrkChjgEXgtXluB8C6PXDFZfUkPC+ffuqPvq+TCh3SFjajf1xXRvGxV0D9drg+mhB2BMnTvT2cWF83e9Lliyp+uh3hI4ddxwy1zc9Vm47pwP+sgQAANCBmyUAAIAO3CwBAAB0ILOU4LIW+rx+yHPYTHFEx+WE1KpVq4r2FVdcUfXZunVr0XaFyvpyCC4Lpc+6L7zwwqrP7t27O5c70TTPo8czU0xyz549VR/Nh7jCbppDyIyLTGE33QaXjcoUOtTilm78ZSZVnY4FCcdFx5sWisyMAZdB07yKuy7pclxhQc0sufE2CpmMy3TLJzlu/2W+Q/Rcy2Sf3NjRwrKuULDm5HS/uwl6hxyrqThJbgZ/WQIAAOjAzRIAAEAHbpYAAAA6cLMEAADQ4YwOeGdCkpk+mSCqBp1dYTAN52VCsS4Mp+/TmakjIt73vvcVbS2CFxGxc+fOon3gwIGqj86GrvvCFTHUEPOKFSuqPlrA0YWGRxXs1H3olqthbd2nGnKOqIPZ7ljp+1wgXpczZ86cqo+GcN1n6bHQoL2bXT5TvPT48eNFe9myZVWfI0eOFO0XX3yx6nMmBHVHRcPZmWKvGuR1hSt13Oo57t7nQsM6Dtw1sC8QHFGPU/2sqRbyH/qjmz6Zorvu3NNxogUoI+pwtPssvQ5dcsklvZ+lY9Bdc7RgbcZ0CHM7/GUJAACgAzdLAAAAHbhZAgAA6HBGZZaGFL3L9HET1+ozYF1OZnLUTHHJTGZJC45F1NkFzZRE1FmiK6+8surzne98p3P9XG5Cj8M73vGOqs+zzz5btF3GZVR0fVz+SLdDJ5p0hTU1a+QmLdXPcvtLswJu/TTH5MaFjlNtZzIlbqJO5YohLl++vGhv2bKl6pMZ7xNlnJOzZjJyfRk0t376missqHkylynU4oKuj+ZK3DHX5bhrqZ57QybfnUjjylC546n5LTdOtI/7LtKskzueehxc1kj7aNsdK3et6jNdJ9TmL0sAAAAduFkCAADowM0SAABAhzMqszSu583uWbLmBTKTGSr3vFc/yz3r1vzAvHnzqj6aD3GTIF588cVFe8OGDb3L0cl3XU0gza/MnTu36qPP2YdmljI5Nd3PLlu0cuXKoq37xh0rfe7vck1ab8vlTPrqm7g+jmYVdEy6zIF+lstUaR83KahmvKZa/Rw1zppPmWVrnTE9z935sGjRoqK9a9euqk9m3Gp+0R1zrbnmtknzM6623FTLJE0UvS5lJlp357hmT90+1s9ydfd27NhRtN13hl6TtS6bG0t9OaeI+lowXccEf1kCAADowM0SAABAB26WAAAAOvTeLDVN8wdN0+xvmmbjKa/NaZrm7qZpNp/8d/Yp/+23mqbZ0jTNM03TvHdcKw4AADARMgHvP4yI/yMi/u9TXvtMRHyrbdvPNk3zmZPtTzdNc0VE/FJEXBkRSyLiL5umWdu27aQkukY1KaILtGogU0Nr7nMyE+lqaDIzUe3SpUt7l+PCg4cPHy7aLoj9oQ99qGhr0cJMkbTLLrus6qOFNN2EnxlDCo+6sOr+/fuLthYVXbBgQfUendTSHU89fi7grX1ciDMz6bKOr74xGlHvC/c5OnZcWFXD7a4oaybkeibI/OhAw9nuGqN0DN59991VHz2HXaFZDW+7CZb1NTdup3qIfzLpvnHnlY4TVxBW97tbjv4Aw/2A5bnnnivablzoDwi0cOr8+fOr9+gk22fymOj9y1Lbtt+LCP1pxoaI+OLJ//3FiPjQKa/f2bbta23K48sFAAAYdUlEQVTbPh8RWyLihhGtKwAAwIQbmlla2Lbt3oiIk//+/f/lWRoRp/6JYNfJ1ypN09zRNM2DTdM8OHAdAAAAxm7UdZZccSH7d7m2bb8QEV+IiGia5sz92x0AAJjWht4s7WuaZnHbtnubplkcEX8f+NgVEadWJFwWEXtOZwVPR2Zix0wfl/UYMhlgpoCXZkj0OXJExOWXX1603US6+vx79uzZVR/Norjn4ZqB0MJks2bNqt6j+QtXzMxlIMZFP8t9tr6mz/3ddmo2y+0/3RdHjx6t+mhxS5epyoxbzQTpZ7vl6nJcQUzddnc+6Ge7oneuMOpUMmQi7kwu0uWRNEOieSSXW9Nj89BDD1V9dHLsm266qeqjy3a5Q82iZPKV7vhO9Ylzx0XHhcuM6jXHnXuaBdScaUSukPG9995btBcvXlz1Wbt2bdHum5g7IuKZZ56pXuszzkmsx2noY7i7IuL2k//79oj42imv/1LTNOc0TbMqItZExP2nt4oAAACTp/cvS03T/MeIuDki5jVNsysi/nVEfDYivtw0zccjYkdE/EJERNu2TzZN8+WIeCoifhQR//1k/RIOAABgFHpvltq2/eXX+U+3vE7/fxMR/+Z0VgoAAGCqoII3AABAh1H/Gm7KG1LE0AXS+oqFZYKfbl00NLxq1aqqjwa6XThOQ5xufTKz0msfndHaLVf3jRY3i/BBxSH02Lh9oX3cfteQta6zC1/qtrtQbqY4nQbg3XLcGOyj66xFNCPqY54pfun2n77mZrKf6oYU1Mv8QMQV49y+fXvnct3M8VoU0IVrN2/eXLTdMddx4Qrf6na5Melew0/o/nPXVndN6evjir3q8XM/KDhy5EjRdsdOl61tdw3asmVL9Vqf6RDmdvjLEgAAQAdulgAAADpwswQAANDhDZdZGiKTIclMkqvvcbkOLR65cuXKqo/mGdzzcC1m5gqKudf6aAYiM9GvmyBSJxIdOulxJrOkr2UKimpmyRXW1M/OFJVz66fvy0yq6va7LkfHhVuu7guXJdNtdxN1ahFDN7nzkAJ2k2lUY9LRcZCZ+FrPPbcuuo9dXko/K5Mvc+uj3IS80zWfcrr0+u+uDZlr1/PPP1+0XXFhzbK57yudqNxdk3Uc6Dq792hx1TMZf1kCAADowM0SAAD/b3v3HjtZeddx/PMN2MWWy95Z9sJ2d4OsywYtkC4oKUTEohLwHyPGJkRiTJMmtkYjxf7R+Id/mBojJGpCKhYjlzS9KDFBLq1JJQHqYklhuS1hd9nfsneW5Za0Eh//mCH8zud898zD7MxvZpb36x/2md8zZ855zpkzD3M+8z1AByZLAAAAHZgsAQAAdCDgXSELeg4KCWehaw9WZqE/D1J6kUqpXfQxC2j687Iwd00hTQ8LDlO4z8O/mWGWW/u8msKQvr98jLNwtAefX3/99VYfD+Vm+8H3eVbEs6ZY5KBjMgse+3ZnQXYPCWfbUFOUsqb45jQZ9pisKfTpRVCPHj3aaGf7yt/T77zzTquPLyc7JpYvX95oP/fcc60+fvzXFFzNDDuGsy47/w/DA/vbtm1r9fExfvXVV1t9/Icb2Y8t/Lzj7/Nsmw4cONBo15xbZxXfLAEAAHRgsgQAANCByRIAAEAHMksJv+6aFQsblPXIrvF79iPLdZx33nmNdnZDTb9GnRUJ9MxDloHwPEPWx7erJjvjsgzOqAxT9C4r1Ofb5TmiLMvjr33kyJFWHy8Y5wU7pXbOJOPHU02uybchy8j5NmT5As+cZceJZ7Oy15qmzFJNYdJR5W1qMhueicuyijXj5cdF9tq+nOwcU1NY089fWaHUmsKLpwLP92TnGFdzfB08eLDRzt5Xvv+yLJsXDPUbLkvtY9DXL3vP+PFVc2PpWc2x8c0SAABAByZLAAAAHZgsAQAAdGCyBAAA0GFmAt41gcya53i4rCbIWLMcD01m4W1/rU2bNrX6rF+/vtHOgp6+Plno2sO0NQHvmgCfLyd7bQ9xLlq0qNVnVEYVFvTtqFmuj5cXfpPax0VNiD4L5XpgNAuc+7J9/bLn1BQd9YB3VmTUj9OszzDv4XGZtpDp8ePHG+0sdJ0Fd50fOzWB6my/eCA444UrP8o8eD2qYLuPcbav/NyV7RcvRJrtX/9xgJ+3s+cMc7zNKr5ZAgAA6MBkCQAAoAOTJQAAgA4zk1mquRmqP1ZzM9ssE+R5iyzr4a+1dOnSRtsLAkrSypUrG+2NGze2+vj6ZEXIvE+2ft4ny1DVFKX05/l219x4ddWqVa0+F110UaO9Y8eOVp+F5Nfa/Rp/dvNYH5tsX9WMlx/bNa+VFQD053k7y47VvLYf29lyVq9e3WhnBTqzmxFPs+z94MdJtk0+plk+alAx0JrioBm/mXN2c2d/f9bkIjN+bGeFGKctGzYM38fZdnpRVs8N1WR5smPJc62HDx9u9fHHsmzbmjVrGu3s/OGP+fFfU3S0Rk0GeBrxzRIAAEAHJksAAAAdmCwBAAB0YLIEAADQYWYSlx48zcJwHmLLCrt5cDJbjof1zjzzzFYfv3t8TYDPg7FZCNy3MwsE14TUPXiXBUb9sZrgnYcbawKuWWh427ZtjfakA94+Xl6sMdsG33/ZGHsht5rlZEFx3zfZPnc1xS59udlre7AzCw3X/DBhmopSjkrN3eWz94ifL3y8snNO9qOWQa+VnZeWLVvWaGd3oK8J3NZs+ywEdwep2U5X83lwzjnnNNorVqxo9fEfwszNzbX6ePB/3759rT7+g4ssmH3w4MHO9fHPRWm49/SsHhN8swQAANCByRIAAEAHJksAAAAdZiaz5NfivWigJF188cWN9tlnn93q49mALAdQU1TOl+PXbrMikJ5XyXIJnmvKrhP7WGS5JpdtQ02exp9Xk4OpuVnxVVdd1WjffffdrT7ZsheKZ9Ky3IkfO9n+dJ6FytQUnMzybp6t8PXLchR+3GY5vwMHDgxcPy9Kmb33svfENPGxGPYmtINyfhnPr2SZNL/RaY3stf3Y8WO9lp8LsmN7VDeUXSg1+7OmEK/3yd4z7vLLL2895p9z9957b6vPY4891mhn5/phip56Bji7Qa+fF7PPkEGflVJ7naexcCXfLAEAAHRgsgQAANCByRIAAECHmcks+bXu9evXt/r4df8sK+M5jposSk12xtfP1yV7rSwf4supqbOUbYNfQ6/JFmWZg0HXkrP18+vz2bVuz2Rk17oXMrM0qIZNNsY165fVG3J+nGbHhY9ztlxfH88Ivf32263n+H7IcldeV6lmm7LjIrvB5yALmV0YZrmjWhd/z/hNtyVpz549A5fj+6bmJr41N9LNllOTv5v2jJKrOd6y49/fw541rblxbXZjdX/tp556qtVn+/btjXb2HvYM4bvvvtvqs3v37kb7kksuabSPHTvWeo5vd825oeY9M+l8UoZvlgAAADowWQIAAOjAZAkAAKADkyUAAIAOMxPwXrt2baO9adOmVp9Vq1Y12l7gUWoHDmsK9WVBXg8Ee4g5K8rnfYYtzjXMcrLArYf8skC88+D48ePHW308yJsFlvfu3dto19wkdJw8YO7h6OwY8DHNCoh6eLZmf2Zhd983WeDcedAzez/4/lyyZEmrz7nnnttoHz16tNXHw6rZcnx9siKLNQVhp0l23Po5pWYbfN9kPxCpKWxYc/7w/ZkF/2v4Omc/5Jh2wxQirdnnvq9q3vdeBFJqFw7Ogtn+2VjzeZCF859++ulG+9prr/3Qy60xbcupxTdLAAAAHZgsAQAAdGCyBAAA0GFmMkueUdq6dWurj19Lzq4t+80Cs+xHzXVPz6v4cv1aszT4ZotSO5sybJ7Acy/Z+vhrZTc69WvbNQU7fTvn5uZafQ4dOtRoD1OwcJQGjXu2flkxP+e5hCx34tfes+yY3/w0W86gIplZHsOXkx37nsU6fPhwq48fX1k+yt9r055HqslEjKpwak2h2WFk2+B5qCxrVHMz7FkrOJkZ1THo4+PtmtfJ3tOeY8qONz9WaopHZplCLz47TDHOGqMa84U+f/DNEgAAQAcmSwAAAB2YLAEAAHRgsgQAANBhZgLeO3bsaLQ3b97c6nPppZc22l58TWqHj7OQmAfmsgJeHmzzAHD2HO+zePHiVh8vOpYF+vyxLGjs6+cBYUl64403Gu0sBO6P1QSNPSiYvbYXpRyVmlBuVvTRQ8w+ptn+9D7Z/vQQrB9/Unt8smPS92f24wAP6vpxkoWuBwVTJWnp0qWNtofzs9fKAqTZvhm0PgsZIh7mtWuKqWb70x/z5dQcSzVqCllmwd2aYo1+DNbccX7ajGqf+7b7OaZmP2SFSGt+ROLnpux97sWDV65c2erjn3s1hWb9fZ+dW2fxuMjwzRIAAEAHJksAAAAdmCwBAAB0mJnM0r59+xrtO+64o9XHb9R5/vnnt/pceOGFjXZ2w1u/Nrt69epWnxUrVjTafq02y5R4DiErPOfXgLO8g+deshyMF8nMsiieocoyJX792/tk2+nPybZhXEUoa/I+Gd8uz+VkBTv9hrLZ/vTMQZZB8MxSthzPVGU5AM+g+fshy7/V3DTal5PlEvwYHKZYnTTZQofDvHbNTbZrlut9shtUHzly5EOuXfuYkNr7Jsu4DFtEcdYMs89rbhg8zE2FH3roodZjV155ZaOd3azb33tZpsrX58CBAwOX4+eh7Ca+fl6sOa8v9A1wR4VvlgAAADowWQIAAOjAZAkAAKADkyUAAIAOMxPw9lBYFgjzO8W/8MILrT47d+5stLOA31lnndVoZ8XCvFDfsmXLOtc3W78sxOkBuSww50XIsm3wsF4WLF63bl2jfdlll7X6ePGyLGDofH2ybci2fVx8fbKAsode/RjIQtcecM1C1z5e2XL8tbLQvO+/rICohy096JkFcv21snCor3NWMNHHL/vhRDbug14r+/HCQhk2iFpTfNP5MZoto6YApsuOSf9BQVZw1WU/cPDn1RTfHMakA8E+7lmo3/nYZPvOl7N///5Wn8cff7zRvuKKK1p9fB8fPny41cf3eRZAP3bsWKPt5yX/QYvUPm6z85uv3yyEuTN8swQAANCByRIAAEAHJksAAAAdZiazNIzs2qhfP82uh3sht6yw2549e05y7Ybn65wVAPRr5FlGyHMv2bVuL0i4Zs2aRjvLc7355puNdlbM7KWXXmo9NoyaLJvLchz+PM/yZBkcL+yW3WDWM0tZ5svzBFm+wftkNycedEPN7Dj2scjyb55R8nye1L5ptecfpHZmLzPJjNKojGIbsvfMMDckzc4Nvpya4pKTLEA56YxLTUbJeXanJrP0yiuvtPr4TcmzdfFzQXb8+bkg259+XPhnRvYcP05PlZvmZvhmCQAAoAOTJQAAgA5MlgAAADowWQIAAOgwMwHvYUJ+Nc8ZVZE5X86oCqnVvHYW+vPHsqCnB4Jfe+21Vh8vhuih5ixo7EUos9f2PjUByEzNmPqys+csWbKk0fYQs/9dageWs9C6h+az1/Ywebbdy5cvb7SzMfUQuhdOrbnDerZ+/lpZkVEPlWY/FsiK7k2zmmMrK7TpIdfsPTLoxxWbN29uPadm/3mQN3tOVpAQ9WrOVd7OQtd+bs9+AOHHxfr161t9du3a1Whn+9ePt2wb/Fy/evXqRtvP/dKp8YOMWnyzBAAA0IHJEgAAQAcmSwAAAB1mJrM0SaPKSw1TQHFUBdlqlpPlYLzomBdby4qQefGybLk1mapR8dfKcmC+nX6NP7vZqBcmzfIhb731VudypXZmKcsEzc3NNdo1+7PmJrk+FlnewXMwWabK81LDZMlOtOxplu1zH9Nsn7tVq1Y12lmx1+xm2INe25crtfNvNcvFB2qyYz6m2Xt6y5YtjfbGjRtbfTZs2NBoZxk5LwLs5xypfX7Nip76Y16UMrvBt6/PqZxh4pslAACADkyWAAAAOjBZAgAA6MBkCQAAoAMB7yFNMqztr10ToM54UNHD2zWvnb2Ob+cw6zJO2X7YvXt3o+2Bav+71C7Slo2fh3uzcLmHqj1QnfXJ9rlvl4cts5C6y0Kcvs5ZqN+L52Vj4eucHRce+h5V4Ltmub6d2b7y4zQ7bod5nz/66KON9hlnnNHq4z8o8FB9tj7PPPNMq8+6desa7Z07dw5cvyyM78YVzh9Vgd8aNT86yNbHg84els6K2rrFixe3HnvyySc710WStm/f3mjX/KCgxu23395oZ2Pu55Ts/FFz3pkFfLMEAADQgckSAABAh4GTpYi4KyIORcSz8x77WkS8EBE/jojvRsTieX+7LSJejogXI+Kz41pxAACAhRCDrv1GxGckvS3pn0spW/uP/Zqk75dS3ouIv5KkUsqtEbFF0n2SPi1ptaRHJf1cKaXzYnZEjOcCNAAAwIk9VUq5bFCngd8slVJ+IOl1e+zhUsr76dEnJK3t//tGSfeXUn5SStkl6WX1Jk4AAAAzaRSZpVskPdj/9xpJe+f9ba7/WEtE/GFEbI+I7dnfAQAApsFJlQ6IiK9Iek/SPe8/lHRLL7GVUu6UdGd/OVyGAwAAU2noyVJE3CzpeknXlA+CT3OS5hfxWCvpteFXDwAAYLKGugwXEddJulXSDaWU+dW3HpB0U0QsiogNki6Q9MOTX00AAIDJGPjNUkTcJ+lqScsjYk7SVyXdJmmRpEf61UyfKKV8vpSyIyK+Kek59S7PfWHQL+EAAACm2cDSAQuyEmSWAADAwhtN6QAAAICPsmm5ke4RSe/fJXJ5v43xYYzHjzEeP8Z4/Bjj8WOMx69rjNfXLGAqLsPNFxHba74Sw/AY4/FjjMePMR4/xnj8GOPxG8UYcxkOAACgA5MlAACADtM4Wbpz0ivwEcAYjx9jPH6M8fgxxuPHGI/fSY/x1GWWAAAApsk0frMEAAAwNZgsAQAAdJiayVJEXBcRL0bEyxHx5Umvz6kgItZFxH9GxPMRsSMivth/fGlEPBIRO/v/XTLpdZ11EXFaRPwoIv6932aMRygiFkfEtyLihf7xfAVjPFoR8cf988SzEXFfRJzBGJ+8iLgrIg5FxLPzHjvhuEbEbf3PwRcj4rOTWevZcoIx/lr/fPHjiPhuRCye97cPPcZTMVmKiNMk/Z2kX5e0RdLvRsSWya7VKeE9SX9SSvl5SZdL+kJ/XL8s6XullAskfa/fxsn5oqTn57UZ49G6XdJ/lFI2S/oF9caaMR6RiFgj6Y8kXVZK2SrpNEk3iTEehW9Ius4eS8e1f36+SdJF/ef8ff/zEd2+ofYYPyJpaynlYkkvqXdP26HHeComS5I+LenlUsorpZSfSrpf0o0TXqeZV0rZX0r5n/6/31LvA2aNemN7d7/b3ZJ+azJreGqIiLWSflPS1+c9zBiPSEScLekzkv5RkkopPy2lvCHGeNROl/SzEXG6pI9Lek2M8UkrpfxA0uv28InG9UZJ95dSflJK2SXpZfU+H9EhG+NSysOllPf6zSckre3/e6gxnpbJ0hpJe+e15/qPYUQi4pOSPiXpSUnnllL2S70JlaSVk1uzU8LfSvozSf837zHGeHQ2Sjos6Z/6lzq/HhGfEGM8MqWUfZL+WtKrkvZLOl5KeViM8bicaFz5LByPWyQ92P/3UGM8LZOlSB6jpsGIRMSZkr4t6UullDcnvT6nkoi4XtKhUspTk16XU9jpki6R9A+llE9JekdcDhqpfmbmRkkbJK2W9ImI+Nxk1+ojic/CEYuIr6gXSbnn/YeSbgPHeFomS3OS1s1rr1XvK2CcpIj4GfUmSveUUr7Tf/hgRJzX//t5kg5Nav1OAb8s6YaI2K3e5eNfiYh/EWM8SnOS5kopT/bb31Jv8sQYj86vStpVSjlcSvlfSd+R9EtijMflROPKZ+EIRcTNkq6X9Hvlg6KSQ43xtEyW/lvSBRGxISI+pl746oEJr9PMi4hQL+fxfCnlb+b96QFJN/f/fbOkf1vodTtVlFJuK6WsLaV8Ur3j9vullM+JMR6ZUsoBSXsj4sL+Q9dIek6M8Si9KunyiPh4/7xxjXoZR8Z4PE40rg9IuikiFkXEBkkXSPrhBNZv5kXEdZJulXRDKeXdeX8aaoynpoJ3RPyGetmP0yTdVUr5ywmv0syLiCsl/ZekZ/RBnubP1cstfVPS+eqdJH+7lOIBRHxIEXG1pD8tpVwfEcvEGI9MRPyiegH6j0l6RdLvq/c/e4zxiETEX0j6HfUuWfxI0h9IOlOM8UmJiPskXS1puaSDkr4q6V91gnHtXza6Rb398KVSyoPJYjHPCcb4NkmLJB3td3uilPL5fv8PPcZTM1kCAACYRtNyGQ4AAGAqMVkCAADowGQJAACgA5MlAACADkyWAAAAOjBZAgAA6MBkCQAAoMP/A3jQPB0zbxe2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "def show(img):\n",
    "    fig, ax=plt.subplots(figsize=(10,10))\n",
    "    npimg = img.numpy()\n",
    "    ax.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')\n",
    "\n",
    "tmp=make_grid(out.cpu().detach(), nrow=4)\n",
    "show(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}