figures.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "Copyright 2018 Google LLC\n",
    "\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "\n",
    "    https://www.apache.org/licenses/LICENSE-2.0\n",
    "\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License.\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figures\n",
    "This notebook contains code for generating the figures and tables from the paper _\"Understanding and Improving Interpolation in Autoencoders via an Adversarial Regularizer\"_.\n",
    "The code is mainly provided as an example and may require modification to be run in a different setting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import scipy.ndimage\n",
    "import lib.eval\n",
    "import collections\n",
    "import tensorflow as tf\n",
    "import glob\n",
    "import lib.utils\n",
    "import all_aes\n",
    "from absl import flags\n",
    "import sys\n",
    "FLAGS = flags.FLAGS\n",
    "FLAGS(['--lr', '0.0001'])\n",
    "\n",
    "import os\n",
    "if not os.path.exists('figures'):\n",
    "    os.makedirs('figures')\n",
    "    \n",
    "def flatten_lines(lines, padding=2):\n",
    "    padding = np.ones((lines.shape[0], padding) + lines.shape[2:])\n",
    "    lines = np.concatenate([padding, lines, padding], 1)\n",
    "    lines = np.concatenate(lines, 0)\n",
    "    return np.transpose(lines, [1, 0] + list(range(2, lines.ndim)))\n",
    "\n",
    "def get_final_value_median(values, steps, N=20):\n",
    "    sorted_steps = np.argsort(steps)\n",
    "    values = np.array(values)[sorted_steps]\n",
    "    return np.median(values[-N:])\n",
    "\n",
    "HEIGHT = 32\n",
    "WIDTH = 32\n",
    "N_LINES = 16\n",
    "\n",
    "START_ANGLE = 5*np.pi/7\n",
    "END_ANGLE = 3*np.pi/2."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example line interpolations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_lines = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "# Cover the space of angles somewhat evenly\n",
    "angles = np.linspace(0, 2*np.pi - np.pi/N_LINES, N_LINES)\n",
    "np.random.shuffle(angles)\n",
    "for n, angle in enumerate(angles):\n",
    "    example_lines[n] = lib.data.draw_line(angle, HEIGHT, WIDTH)[..., 0]\n",
    "\n",
    "fig = plt.figure(figsize=(15, 1))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "ax.imshow(flatten_lines(example_lines), cmap=plt.cm.gray, interpolation='nearest')\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_samples.pdf', aspect='normal')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Correct interpolation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "angles = np.linspace(START_ANGLE, END_ANGLE, N_LINES)\n",
    "\n",
    "for n in range(N_LINES):\n",
    "    line_interpolation[n] = lib.data.draw_line(angles[n], HEIGHT, WIDTH)[..., 0]\n",
    "\n",
    "fig = plt.figure(figsize=(15, 1))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "ax.imshow(flatten_lines(line_interpolation), cmap=plt.cm.gray, interpolation='nearest')\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_correct_interpolation.pdf', aspect='normal')\n",
    "print lib.eval.line_eval(line_interpolation[np.newaxis, ..., np.newaxis])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data-space interpolation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "start_line = lib.data.draw_line(START_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "end_line = lib.data.draw_line(END_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "weights = np.linspace(1, 0, N_LINES)\n",
    "\n",
    "for n in range(N_LINES):\n",
    "    line_interpolation[n] = weights[n]*start_line + (1 - weights[n])*end_line\n",
    "\n",
    "fig = plt.figure(figsize=(15, 1))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "ax.imshow(flatten_lines(line_interpolation), cmap=plt.cm.gray, interpolation='nearest')\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_data_interpolation.pdf', aspect='normal')\n",
    "print lib.eval.line_eval(line_interpolation[np.newaxis, ..., np.newaxis])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Abrupt change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "start_line = lib.data.draw_line(START_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "end_line = lib.data.draw_line(END_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "\n",
    "for n in range(N_LINES):\n",
    "    line_interpolation[n] = start_line if n < N_LINES/2 else end_line\n",
    "\n",
    "fig = plt.figure(figsize=(15, 1))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "ax.imshow(flatten_lines(line_interpolation), cmap=plt.cm.gray, interpolation='nearest')\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_abrupt_interpolation.pdf', aspect='normal')\n",
    "print lib.eval.line_eval(line_interpolation[np.newaxis, ..., np.newaxis])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Overshooting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "\n",
    "angles = np.linspace(START_ANGLE, END_ANGLE - 2*np.pi, N_LINES)\n",
    "\n",
    "for n in range(N_LINES):\n",
    "    line_interpolation[n] = lib.data.draw_line(angles[n], HEIGHT, WIDTH)[..., 0]\n",
    "\n",
    "fig = plt.figure(figsize=(15, 1))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "ax.imshow(flatten_lines(line_interpolation), cmap=plt.cm.gray, interpolation='nearest')\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_overshooting_interpolation.pdf', aspect='normal')\n",
    "print lib.eval.line_eval(line_interpolation[np.newaxis, ..., np.newaxis])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Unrealistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "angles = np.linspace(START_ANGLE, END_ANGLE, N_LINES)\n",
    "blur = np.sin(np.linspace(0, np.pi, N_LINES))\n",
    "\n",
    "for n in range(N_LINES):\n",
    "    line = lib.data.draw_line(angles[n], HEIGHT, WIDTH)[..., 0]\n",
    "    line_interpolation[n] = scipy.ndimage.gaussian_filter(line + np.sqrt(blur[n]), blur[n]*1.5)\n",
    "        \n",
    "fig = plt.figure(figsize=(15, 1))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "ax.imshow(flatten_lines(line_interpolation), cmap=plt.cm.gray, interpolation='nearest', vmin=-1, vmax=1)\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_unrealistic_interpolation.pdf', aspect='normal')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Line results table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULTS_PATH = '/home/craffel/data/dberth/RERUNS/*/lines32'\n",
    "experiments = collections.defaultdict(list)\n",
    "for run_path in glob.glob(RESULTS_PATH):\n",
    "    for path in glob.glob(os.path.join(run_path, '*')):\n",
    "        experiments[os.path.split(path)[-1]].append(os.path.join(path, 'tf', 'summaries'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALGS = collections.OrderedDict([\n",
    "    ('Baseline', 'AEBaseline_depth16_latent16_scales4'),\n",
    "    ('Dropout', 'AEDropout_depth16_dropout0.5_latent16_scales4'),\n",
    "    ('Denoising', 'AEDenoising_depth16_latent16_noise1.0_scales4'),\n",
    "    ('VAE', 'VAE_beta1.0_depth16_latent16_scales4'),\n",
    "    ('AAE', 'AAE_adversary_lr0.0001_depth16_disc_layer_sizes100,100_latent16_scales4'),\n",
    "    ('VQ-VAE', 'AEVQVAE_advdepth16_advweight0.0_beta10.0_depth16_emaTrue_latent16_noise0.0_num_blocks1_num_latents10_num_residuals1_reg0.5_scales3_z_log_size14'),\n",
    "    ('ACAI', 'ARAReg_advdepth16_advweight0.5_depth16_latent16_reg0.2_scales4'),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_results = collections.defaultdict(\n",
    "    lambda: collections.defaultdict(\n",
    "        lambda: collections.defaultdict(\n",
    "            lambda: collections.defaultdict(list))))\n",
    "for experiment_key, experiment_paths in experiments.items():\n",
    "    for n, experiment_path in enumerate(experiment_paths):\n",
    "        print 'Getting results for', experiment_key, n\n",
    "        for events_file in glob.glob(os.path.join(experiment_path, 'events*')):\n",
    "            try:\n",
    "                for e in tf.train.summary_iterator(events_file):\n",
    "                    for v in e.summary.value:\n",
    "                        experiment_results[experiment_key][n][v.tag]['step'].append(e.step)\n",
    "                        experiment_results[experiment_key][n][v.tag]['value'].append(v.simple_value)            \n",
    "            except Exception as e:\n",
    "                print e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_distance = collections.defaultdict(list)\n",
    "mean_smoothness = collections.defaultdict(list)\n",
    "\n",
    "for experiment_name, events_lists in experiment_results.items():\n",
    "    for events in events_lists.values():\n",
    "        mean_distance[experiment_name].append(get_final_value_median(\n",
    "            events['mean_distance_1']['value'], events['mean_distance_1']['step']))\n",
    "        mean_smoothness[experiment_name].append(get_final_value_median(\n",
    "            events['mean_smoothness_1']['value'], events['mean_smoothness_1']['step']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print 'Metric & ' + ' & '.join(ALGS.keys()) + ' \\\\\\\\'\n",
    "print 'Mean Distance ($\\\\times 10^{-3}$) & ' + ' & '.join(\n",
    "    ['{:.2f}$\\pm${:.2f}'.format(np.mean(mean_distance[alg_name])*10**3, np.std(mean_distance[alg_name])*10**3)\n",
    "     for alg_name in ALGS.values()]) + ' \\\\\\\\'\n",
    "print 'Mean Smoothness & ' + ' & '.join(\n",
    "    ['{:.2f}$\\pm${:.2f}'.format(np.mean(mean_smoothness[alg_name]), np.std(mean_smoothness[alg_name]))\n",
    "     for alg_name in ALGS.values()]) + ' \\\\\\\\'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Real line interpolation examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "start_line = lib.data.draw_line(START_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "end_line = lib.data.draw_line(END_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "\n",
    "DATASET = 'lines32'\n",
    "BATCH = 64\n",
    "\n",
    "for alg_name, alg_path in ALGS.items():\n",
    "\n",
    "    ae_path = os.path.join(RESULTS_PATH.replace('*', 'RUN3'), alg_path)\n",
    "    ae, _ = lib.utils.load_ae(ae_path, DATASET, BATCH, all_aes.ALL_AES)\n",
    "    with lib.utils.HookReport.disable():\n",
    "        ae.eval_mode()\n",
    "\n",
    "    input_lines = np.concatenate([\n",
    "        start_line[np.newaxis, ..., np.newaxis],\n",
    "        end_line[np.newaxis, ..., np.newaxis]])\n",
    "    start_latent, end_latent = ae.eval_sess.run(ae.eval_ops.encode, {ae.eval_ops.x: input_lines})\n",
    "    weights = np.linspace(1, 0, N_LINES).reshape(-1, 1, 1, 1)\n",
    "    interped_latents = weights*start_latent[np.newaxis] + (1 - weights)*end_latent[np.newaxis]\n",
    "    output_interp = ae.eval_sess.run(ae.eval_ops.decode, {ae.eval_ops.h: interped_latents})\n",
    "\n",
    "    fig = plt.figure(figsize=(15, 1))\n",
    "    ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "    ax.set_axis_off()\n",
    "    fig.add_axes(ax)\n",
    "    ax.imshow(flatten_lines(output_interp[..., 0]), cmap=plt.cm.gray, interpolation='nearest')\n",
    "    plt.gca().set_axis_off()\n",
    "\n",
    "    plt.savefig('figures/line_{}_example.pdf'.format(alg_name.lower()), aspect='normal')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Real data interpolations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH = 64\n",
    "\n",
    "DBERTH_RESULTS_PATH = '/home/craffel/data/dberth/RERUNS/RUN2'\n",
    "DATASETS_DEPTHS = collections.OrderedDict([('mnist32', 16), ('svhn32', 64), ('celeba32', 64)])\n",
    "LATENTS = [2, 16]\n",
    "ALGS_FORMAT = collections.OrderedDict([\n",
    "    ('Baseline', 'AEBaseline_depth{depth}_latent{latent}_scales3'),\n",
    "    ('Dropout', 'AEDropout_depth{depth}_dropout0.5_latent{latent}_scales3'),\n",
    "    ('Denoising', 'AEDenoising_depth{depth}_latent{latent}_noise1.0_scales3'),\n",
    "    ('VAE', 'VAE_beta1.0_depth{depth}_latent{latent}_scales3'),\n",
    "    ('AAE', 'AAE_adversary_lr0.0001_depth{depth}_disc_layer_sizes100,100_latent{latent}_scales3'),\n",
    "    ('VQ-VAE', 'AEVQVAE_beta10.0_depth{depth}_latent{latent}_num_latents10_run1_scales3_z_log_size14'),\n",
    "    ('ACAI', 'ARAReg_advdepth{depth}_advweight0.5_depth{depth}_latent{latent}_reg0.2_scales3'),\n",
    "])\n",
    "DATASETS_MINS = {'mnist32': -1, 'celeba32': -1.2, 'svhn32': -1}\n",
    "DATASETS_MAXS = {'mnist32': 1, 'celeba32': 1.2, 'svhn32': 1}\n",
    "\n",
    "N_IMAGES_PER_INTERPOLATION = 16\n",
    "N_IMAGES = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def interpolate(sess,\n",
    "                ops,\n",
    "                image_left,\n",
    "                image_right,\n",
    "                dataset_min,\n",
    "                dataset_max,\n",
    "                interpolation=N_IMAGES_PER_INTERPOLATION):\n",
    "    def batched_op(op, op_input, array):\n",
    "        return sess.run(op, feed_dict={op_input: array})\n",
    "\n",
    "    # Interpolations\n",
    "    interpolation_x = np.array([image_left, image_right], 'f')\n",
    "    latent_x = batched_op(ops.encode, ops.x, interpolation_x)\n",
    "    latents = []\n",
    "    for x in range(interpolation):\n",
    "        latents.append((latent_x[:1] * (interpolation - x - 1) +\n",
    "                        latent_x[1:] * x) / float(interpolation - 1))\n",
    "    latents = np.concatenate(latents, axis=0)\n",
    "    interpolation_y = batched_op(ops.decode, ops.h, latents)\n",
    "    interpolation_y = interpolation_y.reshape(\n",
    "        (interpolation, 1) + interpolation_y.shape[1:])\n",
    "    interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)\n",
    "    image_interpolation = lib.utils.images_to_grid(interpolation_y)\n",
    "    padding = np.ones((image_interpolation.shape[0], 2) + image_interpolation.shape[2:])\n",
    "    image = np.concatenate(\n",
    "        [image_left, padding, image_interpolation, padding, image_right],\n",
    "        axis=1)\n",
    "    image = (image - dataset_min)/(dataset_max - dataset_min)\n",
    "    image = np.clip(image, 0, 1)\n",
    "    return image\n",
    "\n",
    "def get_dataset_samples(sess, ops, dataset, batches=100):\n",
    "    batch = FLAGS.batch\n",
    "    with tf.Graph().as_default():\n",
    "        data_in = dataset.make_one_shot_iterator().get_next()\n",
    "        with tf.Session() as sess_new:\n",
    "            images = []\n",
    "            labels = []\n",
    "            while True:\n",
    "                try:\n",
    "                    payload = sess_new.run(data_in)\n",
    "                    images.append(payload['x'])\n",
    "                    assert images[-1].shape[0] == 1\n",
    "                    labels.append(payload['label'])\n",
    "                    if len(images) == batches:\n",
    "                        break\n",
    "                except tf.errors.OutOfRangeError:\n",
    "                    break\n",
    "    images = np.concatenate(images, axis=0)\n",
    "    labels = np.concatenate(labels, axis=0)\n",
    "    latents = [sess.run(ops.encode,\n",
    "                        feed_dict={ops.x: images[p:p + batch]})\n",
    "               for p in range(0, images.shape[0], FLAGS.batch)]\n",
    "    latents = np.concatenate(latents, axis=0)\n",
    "    latents = latents.reshape([latents.shape[0], -1])\n",
    "    return images, latents, labels\n",
    "\n",
    "left_images = collections.defaultdict(lambda: None)\n",
    "right_images = collections.defaultdict(lambda: None)\n",
    "\n",
    "for dataset, depth in DATASETS_DEPTHS.items():\n",
    "    for latent in LATENTS:\n",
    "        for alg_name, alg_format in ALGS_FORMAT.items():\n",
    "            for n in range(N_IMAGES):\n",
    "                output_name = '{}_{}_latent_{}_interpolation_{}'.format(dataset, alg_name.lower(), latent, n + 1)\n",
    "                alg_path = os.path.join(DBERTH_RESULTS_PATH, dataset, alg_format.format(depth=depth, latent=latent))\n",
    "\n",
    "                if 1: # try:\n",
    "                    ae, ds = lib.utils.load_ae(\n",
    "                        alg_path, dataset, BATCH, all_aes.ALL_AES, return_dataset=True)\n",
    "                    with lib.utils.HookReport.disable():\n",
    "                        ae.eval_mode()\n",
    "\n",
    "                    images, latents, labels = get_dataset_samples(ae.eval_sess,\n",
    "                                                                  ae.eval_ops,\n",
    "                                                                  ds.test)\n",
    "                    labels = np.argmax(labels, axis=1)\n",
    "                    if left_images[n] is None:\n",
    "                        left_img_idx = n\n",
    "                        if dataset == 'celeba32':\n",
    "                            right_img_idx = N_IMAGES + n\n",
    "                        else:\n",
    "                            if n < N_IMAGES/2:\n",
    "                                right_img_idx = np.flatnonzero(labels == labels[n])[N_IMAGES + n]\n",
    "                            else:\n",
    "                                right_img_idx = np.flatnonzero(labels != labels[n])[N_IMAGES + n]\n",
    "                        print left_img_idx, labels[left_img_idx]\n",
    "                        print right_img_idx, labels[right_img_idx]\n",
    "                        left_images[n] = images[left_img_idx]\n",
    "                        right_images[n] = images[right_img_idx]\n",
    "                    left_image = left_images[n]\n",
    "                    right_image = right_images[n]\n",
    "                    image = interpolate(ae.eval_sess, ae.eval_ops, left_image, right_image,\n",
    "                                        DATASETS_MINS[dataset], DATASETS_MAXS[dataset])\n",
    "                fig = plt.figure(figsize=(15, 1))\n",
    "                ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "                ax.set_axis_off()\n",
    "                fig.add_axes(ax)\n",
    "                ax.imshow(np.squeeze(image), cmap=plt.cm.gray, interpolation='nearest')\n",
    "                plt.gca().set_axis_off()\n",
    "\n",
    "                plt.savefig('figures/{}.pdf'.format(output_name), aspect='normal')\n",
    "                plt.close()\n",
    "    for n in range(N_IMAGES):\n",
    "        del left_images[n]\n",
    "        del right_images[n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_NAMES = {'mnist32': 'MNIST', 'svhn32': 'SVHN', 'celeba32': 'CelebA'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "output = \"\"\n",
    "\n",
    "for dataset, depth in DATASETS_DEPTHS.items():\n",
    "    for latent in LATENTS:\n",
    "        output += r\"\"\"\n",
    "    \\begin{figure}\n",
    "      \\centering\n",
    "\"\"\"\n",
    "        for n in range(N_IMAGES):\n",
    "            alg_list = collections.OrderedDict()\n",
    "            for alg_name, alg_format in ALGS_FORMAT.items():\n",
    "                figure_name = '{}_{}_latent_{}_interpolation_{}'.format(dataset, alg_name.lower(), latent, n + 1)\n",
    "                alg_list[figure_name] = alg_name\n",
    "                if alg_name == ALGS_FORMAT.keys()[-1]:\n",
    "                    reset = r\"\\addtocounter{{subfigure}}{{-{}}}\".format(len(ALGS_FORMAT))\n",
    "                else:\n",
    "                    reset = \"\"\n",
    "                output += r\"\"\"\n",
    "      \\begin{{subfigure}}[b]{{\\textwidth}}\n",
    "        \\centering\\parbox{{.09\\linewidth}}{{\\vspace{{0.3em}}\\subcaption{{}}\\label{{fig:{figure_name}}}}}\n",
    "        \\parbox{{.75\\linewidth}}{{\\includegraphics[width=\\linewidth]{{figures/{figure_name}.pdf}}}}{reset}\n",
    "      \\end{{subfigure}}\n",
    "\"\"\".format(figure_name=figure_name, reset=reset)\n",
    "                if alg_name == ALGS_FORMAT.keys()[-1]:\n",
    "                    output += r\"\"\"\n",
    "      \\vspace{0.5em}\n",
    "\"\"\"\n",
    "        output += r\"\"\"\n",
    "      \\caption{{Example interpolations on {} with a latent dimensionality of {} for \"\"\".format(\n",
    "            DATASET_NAMES[dataset], latent*16)\n",
    "        output += ', '.join([r'(\\subref{{fig:{}}}) {}'.format(fn, an) for fn, an in alg_list.items()])\n",
    "        output += r\"\"\" autoencoders.}}\n",
    "      \\label{{fig:{}_{}_interpolations}}\n",
    "    \\end{{figure}}\n",
    "\n",
    "\n",
    "\"\"\".format(dataset, latent)\n",
    "\n",
    "print output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VAE line samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULTS_PATH = '/home/craffel/data/autoencoder/results_final/lines32'\n",
    "\n",
    "line_interpolation = np.zeros((N_LINES, HEIGHT, WIDTH))\n",
    "start_line = lib.data.draw_line(START_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "end_line = lib.data.draw_line(END_ANGLE, HEIGHT, WIDTH)[..., 0]\n",
    "\n",
    "DATASET = 'lines32'\n",
    "BATCH = 64\n",
    "\n",
    "ae_path = os.path.join(RESULTS_PATH, 'VAE_beta1.0_depth16_latent16_scales4')\n",
    "ae, _ = lib.utils.load_ae(ae_path, DATASET, BATCH, all_aes.ALL_AES)\n",
    "with lib.utils.HookReport.disable():\n",
    "    ae.eval_mode()\n",
    "\n",
    "random_latents = np.random.standard_normal(size=(16*16, 2, 2, 16))\n",
    "\n",
    "random_images = ae.eval_sess.run(ae.eval_ops.decode, {ae.eval_ops.h: random_latents})\n",
    "\n",
    "fig = plt.figure(figsize=(15, 15))\n",
    "ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "ax.set_axis_off()\n",
    "fig.add_axes(ax)\n",
    "padding = np.ones((2, WIDTH*N_LINES + 4*N_LINES))\n",
    "line_matrix = np.concatenate([\n",
    "    np.concatenate([padding, flatten_lines(random_images[n:n + 16, ..., 0]), padding], axis=0)\n",
    "    for n in range(0, 16*16, 16)], axis=0)\n",
    "ax.imshow(line_matrix, cmap=plt.cm.gray, interpolation='nearest')\n",
    "plt.gca().set_axis_off()\n",
    "\n",
    "plt.savefig('figures/line_vae_samples.pdf'.format(alg_name.lower()), aspect='normal')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single-layer classifier table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_results(results_path, event_key):\n",
    "    experiments = collections.defaultdict(list)\n",
    "    for run_path in glob.glob(results_path):\n",
    "        for path in glob.glob(os.path.join(run_path, '*')):\n",
    "            experiments[os.path.split(path)[-1]].append(os.path.join(path, 'tf', 'summaries'))\n",
    "\n",
    "    experiment_results = collections.defaultdict(\n",
    "        lambda: collections.defaultdict(\n",
    "            lambda: collections.defaultdict(\n",
    "                lambda: collections.defaultdict(list))))\n",
    "    for experiment_key, experiment_paths in experiments.items():\n",
    "        for n, experiment_path in enumerate(experiment_paths):\n",
    "            print 'Getting results for', experiment_key, n\n",
    "            for events_file in glob.glob(os.path.join(experiment_path, 'events*')):\n",
    "                try:\n",
    "                    for e in tf.train.summary_iterator(events_file):\n",
    "                        for v in e.summary.value:\n",
    "                            experiment_results[experiment_key][n][v.tag]['step'].append(e.step)\n",
    "                            experiment_results[experiment_key][n][v.tag]['value'].append(v.simple_value)            \n",
    "                except Exception as e:\n",
    "                    print e\n",
    "\n",
    "    event_values = collections.defaultdict(list)\n",
    "    for experiment_name, events_lists in experiment_results.items():\n",
    "        for events in events_lists.values():\n",
    "            event_values[experiment_name].append(get_final_value_median(\n",
    "                events[event_key]['value'], events[event_key]['step']))\n",
    "    return event_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULTS_PATH = '/home/craffel/data/dberth/RERUNS/*/mnist32'\n",
    "accuracy = get_all_results(RESULTS_PATH, 'latent_accuracy_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALGS = collections.OrderedDict([\n",
    "    ('Baseline', 'AEBaseline_depth16_latent{}_scales3'),\n",
    "    ('Dropout', 'AEDropout_depth16_dropout0.5_latent{}_scales3'),\n",
    "    ('Denoising', 'AEDenoising_depth16_latent{}_noise1.0_scales3'),\n",
    "    ('VAE', 'VAE_beta1.0_depth16_latent{}_scales3'),\n",
    "    ('AAE', 'AAE_adversary_lr0.0001_depth16_disc_layer_sizes100,100_latent{}_scales3'),\n",
    "    ('VQ-VAE', 'AEVQVAE_advdepth16_advweight0.0_beta10.0_depth16_emaTrue_latent{}_noiseFalse_num_blocks1_num_latents10_num_residuals1_reg0.5_scales3_z_log_size14'),\n",
    "    ('ACAI', 'ARAReg_advdepth16_advweight0.5_depth16_latent{}_reg0.2_scales3')])\n",
    "\n",
    "for latent_size in [2, 16]:\n",
    "    print '{} & '.format(latent_size*16) + ' & '.join(\n",
    "        ['{:.2f}$\\pm${:.2f}'.format(\n",
    "            np.mean(accuracy[alg_name.format(latent_size)]),\n",
    "            np.std(accuracy[alg_name.format(latent_size)]))\n",
    "         for alg_name in ALGS.values()]) + ' \\\\\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULTS_PATH = '/home/craffel/data/dberth/RERUNS/*/svhn32'\n",
    "accuracy = get_all_results(RESULTS_PATH, 'latent_accuracy_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALGS = collections.OrderedDict([\n",
    "    ('Baseline', 'AEBaseline_depth64_latent{}_scales3'),\n",
    "    ('Dropout', 'AEDropout_depth64_dropout0.5_latent{}_scales3'),\n",
    "    ('Denoising', 'AEDenoising_depth64_latent{}_noise1.0_scales3'),\n",
    "    ('VAE', 'VAE_beta1.0_depth64_latent{}_scales3'),\n",
    "    ('AAE', 'AAE_adversary_lr0.0001_depth64_disc_layer_sizes100,100_latent{}_scales3'),\n",
    "    ('VQ-VAE', 'AEVQVAE_advdepth16_advweight0.0_beta10.0_depth64_emaTrue_latent{}_noiseFalse_num_blocks1_num_latents10_num_residuals1_reg0.5_scales3_z_log_size14'),\n",
    "    ('ACAI', 'ARAReg_advdepth64_advweight0.5_depth64_latent{}_reg0.2_scales3')])\n",
    "\n",
    "for latent_size in [2, 16]:\n",
    "    print '{} & '.format(latent_size*16) + ' & '.join(\n",
    "        ['{:.2f}$\\pm${:.2f}'.format(\n",
    "            np.mean(accuracy[alg_name.format(latent_size)]),\n",
    "            np.std(accuracy[alg_name.format(latent_size)]))\n",
    "         for alg_name in ALGS.values()]) + ' \\\\\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULTS_PATH = '/home/craffel/data/dberth/RERUNS/*/cifar10'\n",
    "accuracy = get_all_results(RESULTS_PATH, 'latent_accuracy_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALGS = collections.OrderedDict([\n",
    "    ('Baseline', 'AEBaseline_depth64_latent{}_scales3'),\n",
    "    ('Dropout', 'AEDropout_depth64_dropout0.75_latent{}_scales3'),\n",
    "    ('Denoising', 'AEDenoising_depth64_latent{}_noise1.0_scales3'),\n",
    "    ('VAE', 'VAE_beta1.0_depth64_latent{}_scales3'),\n",
    "    ('AAE', 'AAE_adversary_lr0.0001_depth64_disc_layer_sizes100,100_latent{}_scales3'),\n",
    "    ('VQ-VAE', 'AEVQVAE_advdepth16_advweight0.0_beta10.0_depth64_emaTrue_latent{}_noiseFalse_num_blocks1_num_latents10_num_residuals1_reg0.5_scales3_z_log_size14'),\n",
    "    ('ACAI', 'ARAReg_advdepth64_advweight0.5_depth64_latent{}_reg0.2_scales3')])\n",
    "\n",
    "for latent_size in [16, 64]:\n",
    "    print '{} & '.format(latent_size*16) + ' & '.join(\n",
    "        ['{:.2f}$\\pm${:.2f}'.format(\n",
    "            np.mean(accuracy[alg_name.format(latent_size)]),\n",
    "            np.std(accuracy[alg_name.format(latent_size)]))\n",
    "         for alg_name in ALGS.values()]) + ' \\\\\\\\'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}