script_v0.3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow working version: 1.8.0\n",
      "Python version: 3.6.7 |Anaconda, Inc.| (default, Oct 23 2018, 19:16:44) \n",
      "[GCC 7.3.0]\n"
     ]
    }
   ],
   "source": [
    "################################################################################################\n",
    "####\n",
    "####       Solids classificator\n",
    "####       @ NeuroneLab - University of Salerno  - IT\n",
    "####       dott. Francesco Bardozzo    : fbardozzo@unisa.it\n",
    "####       dott. Gioele Ciaparrone     : gciaparrone@unisa.it\n",
    "###        dott. Mattia Delli Priscoli : mdellipriscoli@unisa.it\n",
    "#################################################################################################\n",
    "\n",
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "\n",
    "from __future__ import print_function\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import math\n",
    "from numpy.random import RandomState\n",
    "from random import shuffle\n",
    "from progressbar import *\n",
    "import time\n",
    "import GPUtil\n",
    "\n",
    "np.random.seed(17)\n",
    "\n",
    "# In[2]:\n",
    "#InteractiveSession.close()\n",
    "\n",
    "# Set the Graph\n",
    "g = tf.Graph()\n",
    "\n",
    "# Define the CNN parameters\n",
    "epochs = 10\n",
    "\n",
    "batch_val = 10\n",
    "n_batches = 70\n",
    "#\t\t\"data_dir\":'/home/francis/Scrivania/NeuroImagingProject2018/PythonSource',\n",
    "width = 32\n",
    "height = 32\n",
    "depth = 32\n",
    "exp_size = 700\n",
    "test_size = 300\n",
    "\n",
    "with g.as_default():\n",
    "\tbatch_size = tf.placeholder(tf.int64, name=\"batch\")\n",
    "\tkeep_prob = tf.placeholder(tf.float32, name=\"keepprob\")\n",
    "\n",
    "# In[3]:\n",
    "\n",
    "\n",
    "# Tensorflow working version: 1.5.1\n",
    "print(\"Tensorflow working version: %s\" % tf.__version__)\n",
    "\n",
    "import sys\n",
    "print(\"Python version:\", sys.version)\n",
    "\n",
    "\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Device ID (unmasked): 0\n",
      "Device ID (masked): 0\n",
      "Interactive session on GPU initialized.\n",
      "Number of Epochs: 10\n",
      "Batch size: 10\n",
      "Number of batches for each epoch: 70\n"
     ]
    }
   ],
   "source": [
    "# Set CUDA_DEVICE_ORDER so the IDs assigned by CUDA match those from nvidia-smi\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "\n",
    "# Get the first available GPU\n",
    "DEVICE_ID_LIST = GPUtil.getFirstAvailable()\n",
    "# grab first element from list\n",
    "DEVICE_ID = DEVICE_ID_LIST[0]\n",
    "\n",
    "print(DEVICE_ID)\n",
    "# Set CUDA_VISIBLE_DEVICES to mask out all other GPUs than the first available device id\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(DEVICE_ID)\n",
    "\n",
    "# Since all other GPUs are masked out, the first available GPU will now be identified as GPU:0\n",
    "print('Device ID (unmasked): ' + str(DEVICE_ID))\n",
    "print('Device ID (masked): ' + str(0))\n",
    "with tf.device(\"/GPU:0\"):\n",
    "\tsess = tf.InteractiveSession(graph=g)\n",
    "\tprint(\"Interactive session on GPU initialized.\")\n",
    "\n",
    "# In[4]:\n",
    "\n",
    "\n",
    "print(\"Number of Epochs: %d\" % (epochs))\n",
    "print(\"Batch size: %d\" % (batch_val))\n",
    "print('Number of batches for each epoch: %d' % (exp_size / batch_val))\n",
    "\n",
    "# In[5]:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: Shape of the train  set: (?, 32, 32, 32)\n",
      "y: Shape of the labels set: (?, 1)\n"
     ]
    }
   ],
   "source": [
    "with g.as_default():\n",
    "\tx, y = tf.placeholder(tf.float32, shape=[None, width, height, depth], name='input'), tf.placeholder(tf.float32,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tshape=[None, 1],\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tname='labels')\n",
    "\ttrain_dataset = tf.data.Dataset.from_tensor_slices((x, y)).batch(batch_size).repeat()\n",
    "\ttest_dataset = tf.data.Dataset.from_tensor_slices((x, y)).batch(batch_size)\n",
    "\n",
    "\tprint(\"x: Shape of the train  set: %s\" % (x.shape))\n",
    "\tprint(\"y: Shape of the labels set: %s\" % (y.shape))\n",
    "\n",
    "\titer = tf.data.Iterator.from_structure(train_dataset.output_types, train_dataset.output_shapes)\n",
    "\t# Definition of the iterator, features are ginven in input to the fist convolutional layer.\n",
    "\tfeatures, labels = iter.get_next()\n",
    "\ttrain_init_op = iter.make_initializer(train_dataset)\n",
    "\ttest_init_op = iter.make_initializer(test_dataset)\n",
    "\n",
    "# In[6]:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random generation of training set, with shape: (700, 32, 32, 32)\n",
      "Random generation of labels for trainign set, with shape (700, 1)\n",
      "Random generation of test set, with shape: (300, 32, 32, 32)\n",
      "Random generation of labels for trainign set, with shape (300, 1)\n"
     ]
    }
   ],
   "source": [
    "# Training set random generation:\n",
    "train_set_data = []\n",
    "# rr = RandomState()\n",
    "#Load the spheres\n",
    "for i in range(int(exp_size / 2)):\n",
    "\t# FA_data = rr.uniform(0, 300, size=(width, height, depth))\n",
    "\t# print(FA_data.shape)\n",
    "\tFA_data = np.load(\"./data_solid/%d_s.npy\" % i, encoding = 'bytes')\n",
    "\ttrain_set_data = np.append(train_set_data, np.asarray(FA_data, dtype='float32'))\n",
    "\n",
    "for i in range(int(exp_size / 2), exp_size):\n",
    "\t# FA_data = rr.uniform(0, 300, size=(width, height, depth))\n",
    "\t# print(FA_data.shape)\n",
    "\tFA_data = np.load(\"./data_solid/%d_o.npy\" % i, encoding = 'bytes')\n",
    "\ttrain_set_data = np.append(train_set_data, np.asarray(FA_data, dtype='float32'))\n",
    "\n",
    "# reshape training set\n",
    "train_set_data = train_set_data.reshape(exp_size, width, height, depth)\n",
    "print(\"Random generation of training set, with shape:\", (train_set_data.shape))\n",
    "# label_train = np.random.randint(2, size=(exp_size, 1))\n",
    "# print(\"Random generation of labels for trainign set, with shape\", label_train.shape)\n",
    "label_train = np.load(\"./labels_solid/label_train_set.npy\", encoding = 'bytes').reshape((exp_size,1))\n",
    "print(\"Random generation of labels for trainign set, with shape\", label_train.shape)\n",
    "# Training set\n",
    "\n",
    "\n",
    "indices = np.arange(exp_size)\n",
    "np.random.shuffle(indices)\n",
    "\n",
    "\n",
    "train_set_data = train_set_data[indices]\n",
    "label_train = label_train[indices]\n",
    "\n",
    "train_data = (train_set_data, label_train)\n",
    "\n",
    "# In[7]:\n",
    "\n",
    "\n",
    "# Test set random generation:\n",
    "\n",
    "test_set_data = []\n",
    "# rr = RandomState()\n",
    "for i in range(int(test_size / 2)):\n",
    "\t# FLAGS.exp_size\n",
    "\t# FA_data = rr.uniform(0, 300, size=(width, height, depth))\n",
    "\tFA_data = np.load(\"./test_solid/%d_s.npy\" % i, encoding = 'bytes')\n",
    "\ttest_set_data = np.append(test_set_data, np.asarray(FA_data, dtype='float32'))\n",
    "\n",
    "for i in range(int(test_size / 2), test_size):\n",
    "\t# FLAGS.exp_size\n",
    "\t# FA_data = rr.uniform(0, 300, size=(width, height, depth))\n",
    "\tFA_data = np.load(\"./test_solid/%d_o.npy\" % i, encoding = 'bytes')\n",
    "\ttest_set_data = np.append(test_set_data, np.asarray(FA_data, dtype='float32'))\n",
    "# reshape test set\n",
    "test_set_data = test_set_data.reshape(test_size, width, height, depth)\n",
    "print(\"Random generation of test set, with shape:\", (test_set_data.shape))\n",
    "# label_test = np.random.randint(2, size=(test_size, 1))\n",
    "# print(\"Random generation of labels for trainign set, with shape\", label_test.shape)\n",
    "label_test = np.load(\"./labels_solid/label_test_set.npy\", encoding = 'bytes').reshape((test_size,1))\n",
    "print(\"Random generation of labels for trainign set, with shape\", label_test.shape)\n",
    "# Test set\n",
    "test_data = (test_set_data, label_test)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set_data.dump(\"test_set_data.npy\")\n",
    "label_test.dump(\"label_test.npy\")\n",
    "\n",
    "train_set_data.dump(\"train_set_data.npy\")\n",
    "label_train.dump(\"label_train.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of the input layer: (?, 32, 32, 32, 1)\n"
     ]
    }
   ],
   "source": [
    "# Model function definition:\n",
    "def weight_variable(shape, name):\n",
    "\tinitial = tf.truncated_normal(shape, stddev=0.1, name=name)\n",
    "\treturn tf.Variable(initial)\n",
    "\n",
    "\n",
    "def bias_variable(shape, name):\n",
    "\tinitial = tf.constant(0.1, shape=shape, name=name)\n",
    "\treturn tf.Variable(initial)\n",
    "\n",
    "\n",
    "# Convolution 3D-Tensor\n",
    "def conv3d(x, W, name):\n",
    "\t# stride [1, x_movement, y_movement, z_,movement 1]\n",
    "\treturn tf.nn.conv3d(x, W, strides=[1, 1, 1, 1, 1], padding='SAME', name=name)\n",
    "\n",
    "\n",
    "# Pooling: max pooling over 2x2 blocks\n",
    "def max_pool_2x2(x, name):\n",
    "\t# stride [1, x_movement, y_movement, z_movement, 1]\n",
    "\treturn tf.nn.max_pool3d(x, ksize=[1, 2, 2, 2, 1], strides=[1, 2, 2, 2, 1], padding='SAME', name=name)\n",
    "\n",
    "\n",
    "# Input shape\n",
    "x_image = tf.reshape(features, [-1, width, height, depth, 1], name=\"input_x\")\n",
    "print(\"Shape of the input layer: %s\" % (x_image.shape))  # [n_samples, 28,28,28,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_slices(tensor, name, max_outputs):\n",
    "\tx_min = tf.reduce_min(tensor)\n",
    "\tx_max = tf.reduce_max(tensor)\n",
    "\tweights_0_to_1 = (tensor - x_min) / (x_max - x_min)\n",
    "\tweights_0_to_255_uint8 = tf.image.convert_image_dtype(weights_0_to_1, dtype=tf.uint8)\n",
    "\t# to tf.image_summary format [batch_size, height, width, channels]\n",
    "\tweights_transposed = tf.transpose(weights_0_to_255_uint8, [4, 0, 1, 2, 3])\n",
    "\tfor i in range(32):\n",
    "\t\t# this will display random 3 filters from the 64 in conv1\n",
    "\t\ttf.summary.image(name % i, weights_transposed[i,:,:,:,0:1], max_outputs=max_outputs)\n",
    "\n",
    "## First layer ## downsampling\n",
    "#with g.as_default():\n",
    "    #map_feat = 32\n",
    "    #patch1 = [5, 5, 5, 1, map_feat]\n",
    "\n",
    "    #W_conv1 = weight_variable(patch1, \"W_conv1\")  # patch 5x5x5, in size 1, out size 32\n",
    "    #plot_slices(W_conv1, 'conv1/filters_%d', max_outputs=5)\n",
    "    #b_conv1 = bias_variable([map_feat], \"b_conv1\")\n",
    "    #h_conv1 = tf.nn.relu(conv3d(x_image, W_conv1, name=\"h_conv1_3d\") + b_conv1, name=\"h_conv1\")\n",
    "    # output size 28x28x28x32\n",
    "    #plot_slices(h_conv1, 'conv1/out_%d', max_outputs=32)\n",
    "    #h_pool1 = max_pool_2x2(h_conv1, \"h_pool1\")  # output size 14x14x14x32\n",
    "    #print(h_pool1.get_shape)\n",
    "\n",
    "# In[11]:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method Tensor.get_shape of <tf.Tensor 'h_pool2:0' shape=(?, 16, 16, 16, 32) dtype=float32>>\n",
      "<bound method Tensor.get_shape of <tf.Tensor 'h_pool3:0' shape=(?, 8, 8, 8, 64) dtype=float32>>\n",
      "<bound method Tensor.get_shape of <tf.Tensor 'h_fc1_drop/mul:0' shape=(?, 1024) dtype=float32>>\n",
      "<bound method Tensor.get_shape of <tf.Tensor 'output_layer/BiasAdd:0' shape=(?, 1) dtype=float32>>\n",
      "<bound method Tensor.get_shape of <tf.Tensor 'Sigmoid:0' shape=(?, 1) dtype=float32>>\n"
     ]
    }
   ],
   "source": [
    "# First Layer: Convolutional layer + MaxPooling\n",
    "## conv1 layer ## downsampling\n",
    "with g.as_default():\n",
    "    map_feat = 32\n",
    "    patch2 = [5, 5, 5, 1, map_feat]\n",
    "    W_conv2 = weight_variable(patch2, \"W_conv2\")  # patch 5x5x5, in size 1, out size 32\n",
    "    plot_slices(W_conv2, 'conv2/filters_%d', max_outputs=5)\n",
    "    b_conv2 = bias_variable([map_feat], \"b_conv2\")\n",
    "    h_conv2 = tf.nn.relu(conv3d(x_image, W_conv2, name=\"h_conv2_3d\") + b_conv2,\n",
    "                         name=\"h_conv2\")  # output size 28x28x28x32\n",
    "    h_pool2 = max_pool_2x2(h_conv2, \"h_pool2\")  # output size 14x14x14x32\n",
    "    print(h_pool2.get_shape)\n",
    "\n",
    "# In[12]:\n",
    "\n",
    "\n",
    "# Third Layer: Convolutional Layer + MaxPooling\n",
    "with g.as_default():\n",
    "    map_feat2 = 64\n",
    "    patch2 = [5, 5, 5, map_feat, map_feat2]\n",
    "\n",
    "    W_conv3 = weight_variable(patch2, \"W_conv3\")  # patch 5x5, in size 32, out size 64\n",
    "    b_conv3 = bias_variable([map_feat2], \"b_conv3\")\n",
    "    h_conv3 = tf.nn.relu(conv3d(h_pool2, W_conv3, name=\"h_conv3_3d\") + b_conv3, name=\"h_conv3\")  # output size 14x14x64\n",
    "    h_pool3 = max_pool_2x2(h_conv3, \"h_pool3\")  # output size 7x7x64\n",
    "    print(h_pool3.get_shape)\n",
    "\n",
    "    h_pool3_s = h_pool3.get_shape().as_list()\n",
    "\n",
    "# In[13]:\n",
    "\n",
    "\n",
    "# Fourth Layer: Flattened Layer + Dropout\n",
    "with g.as_default():\n",
    "    map_feat3 = 1024\n",
    "    flat_map = h_pool3_s[1] * h_pool3_s[2] * h_pool3_s[3] * h_pool3_s[4]\n",
    "    patch3 = [flat_map, map_feat3]\n",
    "    W_fc1 = weight_variable(patch3, \"W_fc1\")\n",
    "    b_fc1 = bias_variable([1024], \"b_fc1\")\n",
    "    # [n_samples, 7, 7, 64] ->> [n_samples, 7*7*64]\n",
    "    h_pool3_flat = tf.reshape(h_pool3, [-1, flat_map], name=\"h_pool3_flat\")\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1, name=\"h_pool3_flat_mm\") + b_fc1, name=\"h_fc1\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob, name=\"h_fc1_drop\")\n",
    "\n",
    "    print(h_fc1_drop.get_shape)\n",
    "\n",
    "# prediction = tf.layers.dense(h_fc1_drop, 1, activation=tf.tanh)\n",
    "\n",
    "\n",
    "# In[14]:\n",
    "\n",
    "\n",
    "# Output layer:\n",
    "with g.as_default():\n",
    "    h1 = tf.layers.dense(h_fc1_drop, 1, activation=None, name=\"output_layer\")\n",
    "    print(h1.get_shape)\n",
    "\n",
    "# In[15]:\n",
    "\n",
    "\n",
    "with g.as_default():\n",
    "    cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=h1, name=\"sig_log\")\n",
    "    loss = tf.reduce_mean(tf.cast(cross_entropy, tf.float32), name=\"red_m\")\n",
    "    learning_rate = 0.01\n",
    "    #optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "    #grads = optimizer.compute_gradients(loss)\n",
    "    #train_op = optimizer.apply_gradients(grads)\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "    prediction = tf.nn.sigmoid(h1)\n",
    "    correct_prediction = tf.equal(tf.round(prediction), labels)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    grads = tf.gradients(h1, h_conv3)\n",
    "    print(prediction.get_shape)\n",
    "\n",
    "# In[16]:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "                                                                               \r",
      "\r",
      " N/A% |                                                      | / ETA:  --:--:--"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Acc: 0.90 , Loss: 1792.51 100% |///////////////////| | Time:  0:00:03\n",
      "Epoch: 1, Acc: 1.00 , Loss: 0.73   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 0.6757\n",
      "Test Set Accuracy 0.9133 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 91% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Acc: 1.00 , Loss: 8.08 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 2, Acc: 1.00 , Loss: 0.02   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 0.9629\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 91% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 2, Acc: 1.00 , Loss: 0.49 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 3, Acc: 1.00 , Loss: 0.00   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 0.9986\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 90% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 3, Acc: 1.00 , Loss: 0.09 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 4, Acc: 1.00 , Loss: 0.00   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 90% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 4, Acc: 1.00 , Loss: 0.05 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 5, Acc: 1.00 , Loss: 0.00   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 85% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 5, Acc: 1.00 , Loss: 0.03 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 6, Acc: 1.00 , Loss: 0.00   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 90% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 6, Acc: 1.00 , Loss: 0.02 100% |//////////////////////| | Time:  0:00:02\n",
      " N/A% |                                                      | / ETA:  --:--:--"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 59% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 7, Acc: 1.00 , Loss: 0.02 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 8, Acc: 1.00 , Loss: 0.00   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 91% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 8, Acc: 1.00 , Loss: 0.01 100% |//////////////////////| | Time:  0:00:02\n",
      "Epoch: 9, Acc: 1.00 , Loss: 0.00   1% |                      | - ETA:   0:00:05"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 73% | 24% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 9, Acc: 1.00 , Loss: 0.01 100% |//////////////////////| | Time:  0:00:02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Train Accuracy : 1.0000\n",
      "Test Set Accuracy 0.9967 \n",
      "Utilization GPU\n",
      "| ID | GPU | MEM |\n",
      "------------------\n",
      "|  0 | 90% | 24% |\n",
      "Epoch 9 Completed out of 10 loss: 0.011371974616849911\n",
      "Test Loss: 0.000122\n"
     ]
    }
   ],
   "source": [
    "with g.as_default():\n",
    "    adj_loss = 1\n",
    "    limit = 0.8\n",
    "    summary = tf.summary.merge_all()\n",
    "    with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess:\n",
    "        summary_writer = tf.summary.FileWriter('tensorboard', sess.graph)\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        # initialise iterator with train data\n",
    "        print('Training...')\n",
    "        for i in range(epochs):\n",
    "            sess.run(train_init_op, feed_dict={x: train_data[0], y: train_data[1], batch_size: batch_val})\n",
    "            # compute loss & accuracy\n",
    "            widgets = [FormatLabel(''), ' ', Percentage(), ' ', Bar('/'), ' ', RotatingMarker(), ' ', ETA()]\n",
    "            progressbar = ProgressBar(widgets=widgets, maxval=n_batches)\n",
    "            progressbar.start()\n",
    "            tot_loss = 0\n",
    "            train_acc = 0\n",
    "            for k in range(n_batches):\n",
    "\n",
    "                train_accuracy, _, loss_value , cp, summary_str= sess.run([accuracy, train_op, loss, correct_prediction, summary], feed_dict={keep_prob: 0.5})\n",
    "\n",
    "                #tensorboard        \n",
    "                summary_writer.add_summary(summary_str, i*n_batches + k)\n",
    "                summary_writer.flush()\n",
    "                #train acc\n",
    "                train_acc += train_accuracy\n",
    "                tot_loss += loss_value\n",
    "                widgets[0] = FormatLabel('Epoch: {0}, Acc: {1:.2f} , Loss: {2:.2f}'.format(i, train_accuracy, tot_loss / adj_loss))\n",
    "                progressbar.update(k)\n",
    "            progressbar.finish()\n",
    "            \n",
    "            h_out2, h_out3, g_out = sess.run([h_conv2, h_conv3, grads], feed_dict={keep_prob: 1})\n",
    "            for f,v in zip([\"conv2.npy\", \"conv3.npy\", \"grads.npy\"],[h_out2, h_out3, g_out]):\n",
    "                with open(f, \"wb\") as outfile:\n",
    "                    np.save(outfile, v)\n",
    "\n",
    "            sess.run(test_init_op, feed_dict={x: test_data[0], y: test_data[1], batch_size: batch_val})\n",
    "            test_acc = 0\n",
    "            for k in range(math.ceil(test_size/batch_val)):\n",
    "                test_accuracy, f, c = sess.run([accuracy, features, correct_prediction], feed_dict={keep_prob: 1})\n",
    "                #dump missclassifications in last epochs\n",
    "                for idx in range(c.shape[0]):\n",
    "                    fel = f[idx]\n",
    "                    cel = c[idx]\n",
    "                    if not cel and i > 18:\n",
    "                        print(\"Dumping\")\n",
    "                        fel.dump(\"wrong_pred_%d.npy\" % (np.random.randint(10000)))    \n",
    "                test_acc += test_accuracy\n",
    "            print(\"Avg Train Accuracy : {:.4f}\".format(train_acc / n_batches))\n",
    "            print(\"Test Set Accuracy %.4f \" % (test_acc / (test_size/batch_val)))\n",
    "            print(\"Utilization GPU\")\n",
    "            GPUtil.showUtilization()\n",
    "        #if early_stop >= limit:\n",
    "        #break\n",
    "\n",
    "\n",
    "        print('Epoch', i, 'Completed out of', epochs, 'loss:', tot_loss / adj_loss)\n",
    "        # initialise iterator with test data\n",
    "        sess.run(test_init_op,\n",
    "                 feed_dict={x: test_data[0], y: test_data[1], batch_size: 20, keep_prob: 1})\n",
    "        print('Test Loss: {:4f}'.format(sess.run(loss, feed_dict={keep_prob: 1})))\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['batch',\n",
       " 'keepprob',\n",
       " 'input',\n",
       " 'labels',\n",
       " 'count',\n",
       " 'Iterator',\n",
       " 'IteratorToStringHandle',\n",
       " 'IteratorGetNext',\n",
       " 'TensorSliceDataset',\n",
       " 'BatchDataset',\n",
       " 'RepeatDataset',\n",
       " 'make_initializer',\n",
       " 'TensorSliceDataset_1',\n",
       " 'BatchDataset_1',\n",
       " 'make_initializer_1',\n",
       " 'input_x/shape',\n",
       " 'input_x',\n",
       " 'W_conv2/shape',\n",
       " 'W_conv2/mean',\n",
       " 'W_conv2/stddev',\n",
       " 'W_conv2/TruncatedNormal',\n",
       " 'W_conv2/mul',\n",
       " 'W_conv2',\n",
       " 'Variable',\n",
       " 'Variable/Assign',\n",
       " 'Variable/read',\n",
       " 'Rank',\n",
       " 'range/start',\n",
       " 'range/delta',\n",
       " 'range',\n",
       " 'Min',\n",
       " 'Rank_1',\n",
       " 'range_1/start',\n",
       " 'range_1/delta',\n",
       " 'range_1',\n",
       " 'Max',\n",
       " 'sub',\n",
       " 'sub_1',\n",
       " 'truediv',\n",
       " 'convert_image/Mul/y',\n",
       " 'convert_image/Mul',\n",
       " 'convert_image',\n",
       " 'transpose/perm',\n",
       " 'transpose',\n",
       " 'strided_slice/stack',\n",
       " 'strided_slice/stack_1',\n",
       " 'strided_slice/stack_2',\n",
       " 'strided_slice',\n",
       " 'conv2/filters_0/tag',\n",
       " 'conv2/filters_0',\n",
       " 'strided_slice_1/stack',\n",
       " 'strided_slice_1/stack_1',\n",
       " 'strided_slice_1/stack_2',\n",
       " 'strided_slice_1',\n",
       " 'conv2/filters_1/tag',\n",
       " 'conv2/filters_1',\n",
       " 'strided_slice_2/stack',\n",
       " 'strided_slice_2/stack_1',\n",
       " 'strided_slice_2/stack_2',\n",
       " 'strided_slice_2',\n",
       " 'conv2/filters_2/tag',\n",
       " 'conv2/filters_2',\n",
       " 'strided_slice_3/stack',\n",
       " 'strided_slice_3/stack_1',\n",
       " 'strided_slice_3/stack_2',\n",
       " 'strided_slice_3',\n",
       " 'conv2/filters_3/tag',\n",
       " 'conv2/filters_3',\n",
       " 'strided_slice_4/stack',\n",
       " 'strided_slice_4/stack_1',\n",
       " 'strided_slice_4/stack_2',\n",
       " 'strided_slice_4',\n",
       " 'conv2/filters_4/tag',\n",
       " 'conv2/filters_4',\n",
       " 'strided_slice_5/stack',\n",
       " 'strided_slice_5/stack_1',\n",
       " 'strided_slice_5/stack_2',\n",
       " 'strided_slice_5',\n",
       " 'conv2/filters_5/tag',\n",
       " 'conv2/filters_5',\n",
       " 'strided_slice_6/stack',\n",
       " 'strided_slice_6/stack_1',\n",
       " 'strided_slice_6/stack_2',\n",
       " 'strided_slice_6',\n",
       " 'conv2/filters_6/tag',\n",
       " 'conv2/filters_6',\n",
       " 'strided_slice_7/stack',\n",
       " 'strided_slice_7/stack_1',\n",
       " 'strided_slice_7/stack_2',\n",
       " 'strided_slice_7',\n",
       " 'conv2/filters_7/tag',\n",
       " 'conv2/filters_7',\n",
       " 'strided_slice_8/stack',\n",
       " 'strided_slice_8/stack_1',\n",
       " 'strided_slice_8/stack_2',\n",
       " 'strided_slice_8',\n",
       " 'conv2/filters_8/tag',\n",
       " 'conv2/filters_8',\n",
       " 'strided_slice_9/stack',\n",
       " 'strided_slice_9/stack_1',\n",
       " 'strided_slice_9/stack_2',\n",
       " 'strided_slice_9',\n",
       " 'conv2/filters_9/tag',\n",
       " 'conv2/filters_9',\n",
       " 'strided_slice_10/stack',\n",
       " 'strided_slice_10/stack_1',\n",
       " 'strided_slice_10/stack_2',\n",
       " 'strided_slice_10',\n",
       " 'conv2/filters_10/tag',\n",
       " 'conv2/filters_10',\n",
       " 'strided_slice_11/stack',\n",
       " 'strided_slice_11/stack_1',\n",
       " 'strided_slice_11/stack_2',\n",
       " 'strided_slice_11',\n",
       " 'conv2/filters_11/tag',\n",
       " 'conv2/filters_11',\n",
       " 'strided_slice_12/stack',\n",
       " 'strided_slice_12/stack_1',\n",
       " 'strided_slice_12/stack_2',\n",
       " 'strided_slice_12',\n",
       " 'conv2/filters_12/tag',\n",
       " 'conv2/filters_12',\n",
       " 'strided_slice_13/stack',\n",
       " 'strided_slice_13/stack_1',\n",
       " 'strided_slice_13/stack_2',\n",
       " 'strided_slice_13',\n",
       " 'conv2/filters_13/tag',\n",
       " 'conv2/filters_13',\n",
       " 'strided_slice_14/stack',\n",
       " 'strided_slice_14/stack_1',\n",
       " 'strided_slice_14/stack_2',\n",
       " 'strided_slice_14',\n",
       " 'conv2/filters_14/tag',\n",
       " 'conv2/filters_14',\n",
       " 'strided_slice_15/stack',\n",
       " 'strided_slice_15/stack_1',\n",
       " 'strided_slice_15/stack_2',\n",
       " 'strided_slice_15',\n",
       " 'conv2/filters_15/tag',\n",
       " 'conv2/filters_15',\n",
       " 'strided_slice_16/stack',\n",
       " 'strided_slice_16/stack_1',\n",
       " 'strided_slice_16/stack_2',\n",
       " 'strided_slice_16',\n",
       " 'conv2/filters_16/tag',\n",
       " 'conv2/filters_16',\n",
       " 'strided_slice_17/stack',\n",
       " 'strided_slice_17/stack_1',\n",
       " 'strided_slice_17/stack_2',\n",
       " 'strided_slice_17',\n",
       " 'conv2/filters_17/tag',\n",
       " 'conv2/filters_17',\n",
       " 'strided_slice_18/stack',\n",
       " 'strided_slice_18/stack_1',\n",
       " 'strided_slice_18/stack_2',\n",
       " 'strided_slice_18',\n",
       " 'conv2/filters_18/tag',\n",
       " 'conv2/filters_18',\n",
       " 'strided_slice_19/stack',\n",
       " 'strided_slice_19/stack_1',\n",
       " 'strided_slice_19/stack_2',\n",
       " 'strided_slice_19',\n",
       " 'conv2/filters_19/tag',\n",
       " 'conv2/filters_19',\n",
       " 'strided_slice_20/stack',\n",
       " 'strided_slice_20/stack_1',\n",
       " 'strided_slice_20/stack_2',\n",
       " 'strided_slice_20',\n",
       " 'conv2/filters_20/tag',\n",
       " 'conv2/filters_20',\n",
       " 'strided_slice_21/stack',\n",
       " 'strided_slice_21/stack_1',\n",
       " 'strided_slice_21/stack_2',\n",
       " 'strided_slice_21',\n",
       " 'conv2/filters_21/tag',\n",
       " 'conv2/filters_21',\n",
       " 'strided_slice_22/stack',\n",
       " 'strided_slice_22/stack_1',\n",
       " 'strided_slice_22/stack_2',\n",
       " 'strided_slice_22',\n",
       " 'conv2/filters_22/tag',\n",
       " 'conv2/filters_22',\n",
       " 'strided_slice_23/stack',\n",
       " 'strided_slice_23/stack_1',\n",
       " 'strided_slice_23/stack_2',\n",
       " 'strided_slice_23',\n",
       " 'conv2/filters_23/tag',\n",
       " 'conv2/filters_23',\n",
       " 'strided_slice_24/stack',\n",
       " 'strided_slice_24/stack_1',\n",
       " 'strided_slice_24/stack_2',\n",
       " 'strided_slice_24',\n",
       " 'conv2/filters_24/tag',\n",
       " 'conv2/filters_24',\n",
       " 'strided_slice_25/stack',\n",
       " 'strided_slice_25/stack_1',\n",
       " 'strided_slice_25/stack_2',\n",
       " 'strided_slice_25',\n",
       " 'conv2/filters_25/tag',\n",
       " 'conv2/filters_25',\n",
       " 'strided_slice_26/stack',\n",
       " 'strided_slice_26/stack_1',\n",
       " 'strided_slice_26/stack_2',\n",
       " 'strided_slice_26',\n",
       " 'conv2/filters_26/tag',\n",
       " 'conv2/filters_26',\n",
       " 'strided_slice_27/stack',\n",
       " 'strided_slice_27/stack_1',\n",
       " 'strided_slice_27/stack_2',\n",
       " 'strided_slice_27',\n",
       " 'conv2/filters_27/tag',\n",
       " 'conv2/filters_27',\n",
       " 'strided_slice_28/stack',\n",
       " 'strided_slice_28/stack_1',\n",
       " 'strided_slice_28/stack_2',\n",
       " 'strided_slice_28',\n",
       " 'conv2/filters_28/tag',\n",
       " 'conv2/filters_28',\n",
       " 'strided_slice_29/stack',\n",
       " 'strided_slice_29/stack_1',\n",
       " 'strided_slice_29/stack_2',\n",
       " 'strided_slice_29',\n",
       " 'conv2/filters_29/tag',\n",
       " 'conv2/filters_29',\n",
       " 'strided_slice_30/stack',\n",
       " 'strided_slice_30/stack_1',\n",
       " 'strided_slice_30/stack_2',\n",
       " 'strided_slice_30',\n",
       " 'conv2/filters_30/tag',\n",
       " 'conv2/filters_30',\n",
       " 'strided_slice_31/stack',\n",
       " 'strided_slice_31/stack_1',\n",
       " 'strided_slice_31/stack_2',\n",
       " 'strided_slice_31',\n",
       " 'conv2/filters_31/tag',\n",
       " 'conv2/filters_31',\n",
       " 'b_conv2',\n",
       " 'Variable_1',\n",
       " 'Variable_1/Assign',\n",
       " 'Variable_1/read',\n",
       " 'h_conv2_3d',\n",
       " 'add',\n",
       " 'h_conv2',\n",
       " 'h_pool2',\n",
       " 'W_conv3/shape',\n",
       " 'W_conv3/mean',\n",
       " 'W_conv3/stddev',\n",
       " 'W_conv3/TruncatedNormal',\n",
       " 'W_conv3/mul',\n",
       " 'W_conv3',\n",
       " 'Variable_2',\n",
       " 'Variable_2/Assign',\n",
       " 'Variable_2/read',\n",
       " 'b_conv3',\n",
       " 'Variable_3',\n",
       " 'Variable_3/Assign',\n",
       " 'Variable_3/read',\n",
       " 'h_conv3_3d',\n",
       " 'add_1',\n",
       " 'h_conv3',\n",
       " 'h_pool3',\n",
       " 'W_fc1/shape',\n",
       " 'W_fc1/mean',\n",
       " 'W_fc1/stddev',\n",
       " 'W_fc1/TruncatedNormal',\n",
       " 'W_fc1/mul',\n",
       " 'W_fc1',\n",
       " 'Variable_4',\n",
       " 'Variable_4/Assign',\n",
       " 'Variable_4/read',\n",
       " 'b_fc1',\n",
       " 'Variable_5',\n",
       " 'Variable_5/Assign',\n",
       " 'Variable_5/read',\n",
       " 'h_pool3_flat/shape',\n",
       " 'h_pool3_flat',\n",
       " 'h_pool3_flat_mm',\n",
       " 'add_2',\n",
       " 'h_fc1',\n",
       " 'h_fc1_drop/Shape',\n",
       " 'h_fc1_drop/random_uniform/min',\n",
       " 'h_fc1_drop/random_uniform/max',\n",
       " 'h_fc1_drop/random_uniform/RandomUniform',\n",
       " 'h_fc1_drop/random_uniform/sub',\n",
       " 'h_fc1_drop/random_uniform/mul',\n",
       " 'h_fc1_drop/random_uniform',\n",
       " 'h_fc1_drop/add',\n",
       " 'h_fc1_drop/Floor',\n",
       " 'h_fc1_drop/div',\n",
       " 'h_fc1_drop/mul',\n",
       " 'output_layer/kernel/Initializer/random_uniform/shape',\n",
       " 'output_layer/kernel/Initializer/random_uniform/min',\n",
       " 'output_layer/kernel/Initializer/random_uniform/max',\n",
       " 'output_layer/kernel/Initializer/random_uniform/RandomUniform',\n",
       " 'output_layer/kernel/Initializer/random_uniform/sub',\n",
       " 'output_layer/kernel/Initializer/random_uniform/mul',\n",
       " 'output_layer/kernel/Initializer/random_uniform',\n",
       " 'output_layer/kernel',\n",
       " 'output_layer/kernel/Assign',\n",
       " 'output_layer/kernel/read',\n",
       " 'output_layer/bias/Initializer/zeros',\n",
       " 'output_layer/bias',\n",
       " 'output_layer/bias/Assign',\n",
       " 'output_layer/bias/read',\n",
       " 'output_layer/MatMul',\n",
       " 'output_layer/BiasAdd',\n",
       " 'sig_log/zeros_like',\n",
       " 'sig_log/GreaterEqual',\n",
       " 'sig_log/Select',\n",
       " 'sig_log/Neg',\n",
       " 'sig_log/Select_1',\n",
       " 'sig_log/mul',\n",
       " 'sig_log/sub',\n",
       " 'sig_log/Exp',\n",
       " 'sig_log/Log1p',\n",
       " 'sig_log',\n",
       " 'Const',\n",
       " 'red_m',\n",
       " 'gradients/Shape',\n",
       " 'gradients/grad_ys_0',\n",
       " 'gradients/Fill',\n",
       " 'gradients/red_m_grad/Reshape/shape',\n",
       " 'gradients/red_m_grad/Reshape',\n",
       " 'gradients/red_m_grad/Shape',\n",
       " 'gradients/red_m_grad/Tile',\n",
       " 'gradients/red_m_grad/Shape_1',\n",
       " 'gradients/red_m_grad/Shape_2',\n",
       " 'gradients/red_m_grad/Const',\n",
       " 'gradients/red_m_grad/Prod',\n",
       " 'gradients/red_m_grad/Const_1',\n",
       " 'gradients/red_m_grad/Prod_1',\n",
       " 'gradients/red_m_grad/Maximum/y',\n",
       " 'gradients/red_m_grad/Maximum',\n",
       " 'gradients/red_m_grad/floordiv',\n",
       " 'gradients/red_m_grad/Cast',\n",
       " 'gradients/red_m_grad/truediv',\n",
       " 'gradients/sig_log_grad/Shape',\n",
       " 'gradients/sig_log_grad/Shape_1',\n",
       " 'gradients/sig_log_grad/BroadcastGradientArgs',\n",
       " 'gradients/sig_log_grad/Sum',\n",
       " 'gradients/sig_log_grad/Reshape',\n",
       " 'gradients/sig_log_grad/Sum_1',\n",
       " 'gradients/sig_log_grad/Reshape_1',\n",
       " 'gradients/sig_log_grad/tuple/group_deps',\n",
       " 'gradients/sig_log_grad/tuple/control_dependency',\n",
       " 'gradients/sig_log_grad/tuple/control_dependency_1',\n",
       " 'gradients/sig_log/sub_grad/Shape',\n",
       " 'gradients/sig_log/sub_grad/Shape_1',\n",
       " 'gradients/sig_log/sub_grad/BroadcastGradientArgs',\n",
       " 'gradients/sig_log/sub_grad/Sum',\n",
       " 'gradients/sig_log/sub_grad/Reshape',\n",
       " 'gradients/sig_log/sub_grad/Sum_1',\n",
       " 'gradients/sig_log/sub_grad/Neg',\n",
       " 'gradients/sig_log/sub_grad/Reshape_1',\n",
       " 'gradients/sig_log/sub_grad/tuple/group_deps',\n",
       " 'gradients/sig_log/sub_grad/tuple/control_dependency',\n",
       " 'gradients/sig_log/sub_grad/tuple/control_dependency_1',\n",
       " 'gradients/sig_log/Log1p_grad/add/x',\n",
       " 'gradients/sig_log/Log1p_grad/add',\n",
       " 'gradients/sig_log/Log1p_grad/Reciprocal',\n",
       " 'gradients/sig_log/Log1p_grad/mul',\n",
       " 'gradients/sig_log/Select_grad/zeros_like',\n",
       " 'gradients/sig_log/Select_grad/Select',\n",
       " 'gradients/sig_log/Select_grad/Select_1',\n",
       " 'gradients/sig_log/Select_grad/tuple/group_deps',\n",
       " 'gradients/sig_log/Select_grad/tuple/control_dependency',\n",
       " 'gradients/sig_log/Select_grad/tuple/control_dependency_1',\n",
       " 'gradients/sig_log/mul_grad/Shape',\n",
       " 'gradients/sig_log/mul_grad/Shape_1',\n",
       " 'gradients/sig_log/mul_grad/BroadcastGradientArgs',\n",
       " 'gradients/sig_log/mul_grad/Mul',\n",
       " 'gradients/sig_log/mul_grad/Sum',\n",
       " 'gradients/sig_log/mul_grad/Reshape',\n",
       " 'gradients/sig_log/mul_grad/Mul_1',\n",
       " 'gradients/sig_log/mul_grad/Sum_1',\n",
       " 'gradients/sig_log/mul_grad/Reshape_1',\n",
       " 'gradients/sig_log/mul_grad/tuple/group_deps',\n",
       " 'gradients/sig_log/mul_grad/tuple/control_dependency',\n",
       " 'gradients/sig_log/mul_grad/tuple/control_dependency_1',\n",
       " 'gradients/sig_log/Exp_grad/mul',\n",
       " 'gradients/sig_log/Select_1_grad/zeros_like',\n",
       " 'gradients/sig_log/Select_1_grad/Select',\n",
       " 'gradients/sig_log/Select_1_grad/Select_1',\n",
       " 'gradients/sig_log/Select_1_grad/tuple/group_deps',\n",
       " 'gradients/sig_log/Select_1_grad/tuple/control_dependency',\n",
       " 'gradients/sig_log/Select_1_grad/tuple/control_dependency_1',\n",
       " 'gradients/sig_log/Neg_grad/Neg',\n",
       " 'gradients/AddN',\n",
       " 'gradients/output_layer/BiasAdd_grad/BiasAddGrad',\n",
       " 'gradients/output_layer/BiasAdd_grad/tuple/group_deps',\n",
       " 'gradients/output_layer/BiasAdd_grad/tuple/control_dependency',\n",
       " 'gradients/output_layer/BiasAdd_grad/tuple/control_dependency_1',\n",
       " 'gradients/output_layer/MatMul_grad/MatMul',\n",
       " 'gradients/output_layer/MatMul_grad/MatMul_1',\n",
       " 'gradients/output_layer/MatMul_grad/tuple/group_deps',\n",
       " 'gradients/output_layer/MatMul_grad/tuple/control_dependency',\n",
       " 'gradients/output_layer/MatMul_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_fc1_drop/mul_grad/Shape',\n",
       " 'gradients/h_fc1_drop/mul_grad/Shape_1',\n",
       " 'gradients/h_fc1_drop/mul_grad/BroadcastGradientArgs',\n",
       " 'gradients/h_fc1_drop/mul_grad/Mul',\n",
       " 'gradients/h_fc1_drop/mul_grad/Sum',\n",
       " 'gradients/h_fc1_drop/mul_grad/Reshape',\n",
       " 'gradients/h_fc1_drop/mul_grad/Mul_1',\n",
       " 'gradients/h_fc1_drop/mul_grad/Sum_1',\n",
       " 'gradients/h_fc1_drop/mul_grad/Reshape_1',\n",
       " 'gradients/h_fc1_drop/mul_grad/tuple/group_deps',\n",
       " 'gradients/h_fc1_drop/mul_grad/tuple/control_dependency',\n",
       " 'gradients/h_fc1_drop/mul_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_fc1_drop/div_grad/Shape',\n",
       " 'gradients/h_fc1_drop/div_grad/Shape_1',\n",
       " 'gradients/h_fc1_drop/div_grad/BroadcastGradientArgs',\n",
       " 'gradients/h_fc1_drop/div_grad/RealDiv',\n",
       " 'gradients/h_fc1_drop/div_grad/Sum',\n",
       " 'gradients/h_fc1_drop/div_grad/Reshape',\n",
       " 'gradients/h_fc1_drop/div_grad/Neg',\n",
       " 'gradients/h_fc1_drop/div_grad/RealDiv_1',\n",
       " 'gradients/h_fc1_drop/div_grad/RealDiv_2',\n",
       " 'gradients/h_fc1_drop/div_grad/mul',\n",
       " 'gradients/h_fc1_drop/div_grad/Sum_1',\n",
       " 'gradients/h_fc1_drop/div_grad/Reshape_1',\n",
       " 'gradients/h_fc1_drop/div_grad/tuple/group_deps',\n",
       " 'gradients/h_fc1_drop/div_grad/tuple/control_dependency',\n",
       " 'gradients/h_fc1_drop/div_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_fc1_grad/ReluGrad',\n",
       " 'gradients/add_2_grad/Shape',\n",
       " 'gradients/add_2_grad/Shape_1',\n",
       " 'gradients/add_2_grad/BroadcastGradientArgs',\n",
       " 'gradients/add_2_grad/Sum',\n",
       " 'gradients/add_2_grad/Reshape',\n",
       " 'gradients/add_2_grad/Sum_1',\n",
       " 'gradients/add_2_grad/Reshape_1',\n",
       " 'gradients/add_2_grad/tuple/group_deps',\n",
       " 'gradients/add_2_grad/tuple/control_dependency',\n",
       " 'gradients/add_2_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_pool3_flat_mm_grad/MatMul',\n",
       " 'gradients/h_pool3_flat_mm_grad/MatMul_1',\n",
       " 'gradients/h_pool3_flat_mm_grad/tuple/group_deps',\n",
       " 'gradients/h_pool3_flat_mm_grad/tuple/control_dependency',\n",
       " 'gradients/h_pool3_flat_mm_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_pool3_flat_grad/Shape',\n",
       " 'gradients/h_pool3_flat_grad/Reshape',\n",
       " 'gradients/h_pool3_grad/MaxPool3DGrad',\n",
       " 'gradients/h_conv3_grad/ReluGrad',\n",
       " 'gradients/add_1_grad/Shape',\n",
       " 'gradients/add_1_grad/Shape_1',\n",
       " 'gradients/add_1_grad/BroadcastGradientArgs',\n",
       " 'gradients/add_1_grad/Sum',\n",
       " 'gradients/add_1_grad/Reshape',\n",
       " 'gradients/add_1_grad/Sum_1',\n",
       " 'gradients/add_1_grad/Reshape_1',\n",
       " 'gradients/add_1_grad/tuple/group_deps',\n",
       " 'gradients/add_1_grad/tuple/control_dependency',\n",
       " 'gradients/add_1_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_conv3_3d_grad/Shape',\n",
       " 'gradients/h_conv3_3d_grad/Conv3DBackpropInputV2',\n",
       " 'gradients/h_conv3_3d_grad/Shape_1',\n",
       " 'gradients/h_conv3_3d_grad/Conv3DBackpropFilterV2',\n",
       " 'gradients/h_conv3_3d_grad/tuple/group_deps',\n",
       " 'gradients/h_conv3_3d_grad/tuple/control_dependency',\n",
       " 'gradients/h_conv3_3d_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_pool2_grad/MaxPool3DGrad',\n",
       " 'gradients/h_conv2_grad/ReluGrad',\n",
       " 'gradients/add_grad/Shape',\n",
       " 'gradients/add_grad/Shape_1',\n",
       " 'gradients/add_grad/BroadcastGradientArgs',\n",
       " 'gradients/add_grad/Sum',\n",
       " 'gradients/add_grad/Reshape',\n",
       " 'gradients/add_grad/Sum_1',\n",
       " 'gradients/add_grad/Reshape_1',\n",
       " 'gradients/add_grad/tuple/group_deps',\n",
       " 'gradients/add_grad/tuple/control_dependency',\n",
       " 'gradients/add_grad/tuple/control_dependency_1',\n",
       " 'gradients/h_conv2_3d_grad/Shape',\n",
       " 'gradients/h_conv2_3d_grad/Conv3DBackpropInputV2',\n",
       " 'gradients/h_conv2_3d_grad/Shape_1',\n",
       " 'gradients/h_conv2_3d_grad/Conv3DBackpropFilterV2',\n",
       " 'gradients/h_conv2_3d_grad/tuple/group_deps',\n",
       " 'gradients/h_conv2_3d_grad/tuple/control_dependency',\n",
       " 'gradients/h_conv2_3d_grad/tuple/control_dependency_1',\n",
       " 'beta1_power/initial_value',\n",
       " 'beta1_power',\n",
       " 'beta1_power/Assign',\n",
       " 'beta1_power/read',\n",
       " 'beta2_power/initial_value',\n",
       " 'beta2_power',\n",
       " 'beta2_power/Assign',\n",
       " 'beta2_power/read',\n",
       " 'Variable/Adam/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable/Adam/Initializer/zeros/Const',\n",
       " 'Variable/Adam/Initializer/zeros',\n",
       " 'Variable/Adam',\n",
       " 'Variable/Adam/Assign',\n",
       " 'Variable/Adam/read',\n",
       " 'Variable/Adam_1/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable/Adam_1/Initializer/zeros/Const',\n",
       " 'Variable/Adam_1/Initializer/zeros',\n",
       " 'Variable/Adam_1',\n",
       " 'Variable/Adam_1/Assign',\n",
       " 'Variable/Adam_1/read',\n",
       " 'Variable_1/Adam/Initializer/zeros',\n",
       " 'Variable_1/Adam',\n",
       " 'Variable_1/Adam/Assign',\n",
       " 'Variable_1/Adam/read',\n",
       " 'Variable_1/Adam_1/Initializer/zeros',\n",
       " 'Variable_1/Adam_1',\n",
       " 'Variable_1/Adam_1/Assign',\n",
       " 'Variable_1/Adam_1/read',\n",
       " 'Variable_2/Adam/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable_2/Adam/Initializer/zeros/Const',\n",
       " 'Variable_2/Adam/Initializer/zeros',\n",
       " 'Variable_2/Adam',\n",
       " 'Variable_2/Adam/Assign',\n",
       " 'Variable_2/Adam/read',\n",
       " 'Variable_2/Adam_1/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable_2/Adam_1/Initializer/zeros/Const',\n",
       " 'Variable_2/Adam_1/Initializer/zeros',\n",
       " 'Variable_2/Adam_1',\n",
       " 'Variable_2/Adam_1/Assign',\n",
       " 'Variable_2/Adam_1/read',\n",
       " 'Variable_3/Adam/Initializer/zeros',\n",
       " 'Variable_3/Adam',\n",
       " 'Variable_3/Adam/Assign',\n",
       " 'Variable_3/Adam/read',\n",
       " 'Variable_3/Adam_1/Initializer/zeros',\n",
       " 'Variable_3/Adam_1',\n",
       " 'Variable_3/Adam_1/Assign',\n",
       " 'Variable_3/Adam_1/read',\n",
       " 'Variable_4/Adam/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable_4/Adam/Initializer/zeros/Const',\n",
       " 'Variable_4/Adam/Initializer/zeros',\n",
       " 'Variable_4/Adam',\n",
       " 'Variable_4/Adam/Assign',\n",
       " 'Variable_4/Adam/read',\n",
       " 'Variable_4/Adam_1/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable_4/Adam_1/Initializer/zeros/Const',\n",
       " 'Variable_4/Adam_1/Initializer/zeros',\n",
       " 'Variable_4/Adam_1',\n",
       " 'Variable_4/Adam_1/Assign',\n",
       " 'Variable_4/Adam_1/read',\n",
       " 'Variable_5/Adam/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable_5/Adam/Initializer/zeros/Const',\n",
       " 'Variable_5/Adam/Initializer/zeros',\n",
       " 'Variable_5/Adam',\n",
       " 'Variable_5/Adam/Assign',\n",
       " 'Variable_5/Adam/read',\n",
       " 'Variable_5/Adam_1/Initializer/zeros/shape_as_tensor',\n",
       " 'Variable_5/Adam_1/Initializer/zeros/Const',\n",
       " 'Variable_5/Adam_1/Initializer/zeros',\n",
       " 'Variable_5/Adam_1',\n",
       " 'Variable_5/Adam_1/Assign',\n",
       " 'Variable_5/Adam_1/read',\n",
       " 'output_layer/kernel/Adam/Initializer/zeros/shape_as_tensor',\n",
       " 'output_layer/kernel/Adam/Initializer/zeros/Const',\n",
       " 'output_layer/kernel/Adam/Initializer/zeros',\n",
       " 'output_layer/kernel/Adam',\n",
       " 'output_layer/kernel/Adam/Assign',\n",
       " 'output_layer/kernel/Adam/read',\n",
       " 'output_layer/kernel/Adam_1/Initializer/zeros/shape_as_tensor',\n",
       " 'output_layer/kernel/Adam_1/Initializer/zeros/Const',\n",
       " 'output_layer/kernel/Adam_1/Initializer/zeros',\n",
       " 'output_layer/kernel/Adam_1',\n",
       " 'output_layer/kernel/Adam_1/Assign',\n",
       " 'output_layer/kernel/Adam_1/read',\n",
       " 'output_layer/bias/Adam/Initializer/zeros',\n",
       " 'output_layer/bias/Adam',\n",
       " 'output_layer/bias/Adam/Assign',\n",
       " 'output_layer/bias/Adam/read',\n",
       " 'output_layer/bias/Adam_1/Initializer/zeros',\n",
       " 'output_layer/bias/Adam_1',\n",
       " 'output_layer/bias/Adam_1/Assign',\n",
       " 'output_layer/bias/Adam_1/read',\n",
       " 'Adam/learning_rate',\n",
       " 'Adam/beta1',\n",
       " 'Adam/beta2',\n",
       " 'Adam/epsilon',\n",
       " 'Adam/update_Variable/ApplyAdam',\n",
       " 'Adam/update_Variable_1/ApplyAdam',\n",
       " 'Adam/update_Variable_2/ApplyAdam',\n",
       " 'Adam/update_Variable_3/ApplyAdam',\n",
       " 'Adam/update_Variable_4/ApplyAdam',\n",
       " 'Adam/update_Variable_5/ApplyAdam',\n",
       " 'Adam/update_output_layer/kernel/ApplyAdam',\n",
       " 'Adam/update_output_layer/bias/ApplyAdam',\n",
       " 'Adam/mul',\n",
       " 'Adam/Assign',\n",
       " 'Adam/mul_1',\n",
       " 'Adam/Assign_1',\n",
       " 'Adam',\n",
       " 'Sigmoid',\n",
       " 'Round',\n",
       " 'Equal',\n",
       " 'Cast_1',\n",
       " 'Const_1',\n",
       " 'Mean',\n",
       " 'gradients_1/Shape',\n",
       " 'gradients_1/grad_ys_0',\n",
       " 'gradients_1/Fill',\n",
       " 'gradients_1/output_layer/BiasAdd_grad/BiasAddGrad',\n",
       " 'gradients_1/output_layer/MatMul_grad/MatMul',\n",
       " 'gradients_1/output_layer/MatMul_grad/MatMul_1',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Shape',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Shape_1',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/BroadcastGradientArgs',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Mul',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Sum',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Reshape',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Mul_1',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Sum_1',\n",
       " 'gradients_1/h_fc1_drop/mul_grad/Reshape_1',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Shape',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Shape_1',\n",
       " 'gradients_1/h_fc1_drop/div_grad/BroadcastGradientArgs',\n",
       " 'gradients_1/h_fc1_drop/div_grad/RealDiv',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Sum',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Reshape',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Neg',\n",
       " 'gradients_1/h_fc1_drop/div_grad/RealDiv_1',\n",
       " 'gradients_1/h_fc1_drop/div_grad/RealDiv_2',\n",
       " 'gradients_1/h_fc1_drop/div_grad/mul',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Sum_1',\n",
       " 'gradients_1/h_fc1_drop/div_grad/Reshape_1',\n",
       " 'gradients_1/h_fc1_grad/ReluGrad',\n",
       " 'gradients_1/add_2_grad/Shape',\n",
       " 'gradients_1/add_2_grad/Shape_1',\n",
       " 'gradients_1/add_2_grad/BroadcastGradientArgs',\n",
       " 'gradients_1/add_2_grad/Sum',\n",
       " 'gradients_1/add_2_grad/Reshape',\n",
       " 'gradients_1/add_2_grad/Sum_1',\n",
       " 'gradients_1/add_2_grad/Reshape_1',\n",
       " 'gradients_1/h_pool3_flat_mm_grad/MatMul',\n",
       " 'gradients_1/h_pool3_flat_mm_grad/MatMul_1',\n",
       " 'gradients_1/h_pool3_flat_grad/Shape',\n",
       " 'gradients_1/h_pool3_flat_grad/Reshape',\n",
       " 'gradients_1/h_pool3_grad/MaxPool3DGrad',\n",
       " 'Merge/MergeSummary',\n",
       " 'init']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[n.name for n in tf.get_default_graph().as_graph_def().node]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (myenv)",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}