cifar_binary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "CRXgJFHTEOPI",
    "outputId": "b7a52b5a-88aa-4dd1-eab7-cbfd92e75147"
   },
   "outputs": [],
   "source": [
    "#!pip uninstall tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 853
    },
    "colab_type": "code",
    "id": "0PW6ikG-ERZV",
    "outputId": "60475b27-a9d9-4247-b079-4f7e68135ff3"
   },
   "outputs": [],
   "source": [
    "#!pip install tensorflow-gpu==2.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "a80IPFtiYlm-",
    "outputId": "7e8cac94-a7ba-4019-a287-9e5a7d56e29e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.compat.v1 import enable_eager_execution\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import tensorflow_probability as tfp\n",
    "from tensorflow.keras import layers\n",
    "import time\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import gridspec\n",
    "import matplotlib.patches as mpatches\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import random\n",
    "from sklearn.metrics import confusion_matrix\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.client.session.Session at 0x1d1a9531828>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.compat.v1.Session(config=config)\n",
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.95\n",
    "tf.compat.v1.Session(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "B1hkHlXiQ1io"
   },
   "source": [
    "## 參數設定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VW-G91MqQ1ip"
   },
   "outputs": [],
   "source": [
    "flatten_size = 32 * 32 * 3\n",
    "img_size = 32\n",
    "img_channel = 3\n",
    "regular = 1e-5\n",
    "\n",
    "z_dim = 128\n",
    "h_dim = [32, 64, 128]\n",
    "objective = 'deep-SVDD'     #'  'soft-boundary'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mLtweCA9Q1if"
   },
   "source": [
    "## 匯入資料（MNIST）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = np.array(['airplane',\n",
    "'automobile',\n",
    "'bird',\n",
    "'cat',\n",
    "'deer',\n",
    "'dog',\n",
    "'frog',\n",
    "'horse',\n",
    "'ship',\n",
    "'truck'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "location  = 'E:/咪聽/SVDD/deep SVDD/checkpoints/'\n",
    "location  = 'D:/7107018014/checkpoint/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "F-rxo8dLYlnB",
    "outputId": "a90ed8ac-dbc9-4372-96e3-2269366f9f7c"
   },
   "outputs": [],
   "source": [
    "def load_data(label_num, batch_size = 200, outlier_nu = 0.1):\n",
    "    # 0: normal, 1: outlier\n",
    "    \n",
    "    x_train = np.load('D:/7107018014/cifar10/{}/{}_data{}_batch{}.npy'.format(label_num,'train',label_num, 0))\n",
    "    y_train = np.load('D:/7107018014/cifar10/{}/{}_label{}_batch{}.npy'.format(label_num,'train',label_num, 0))\n",
    "    x_train = np.transpose(x_train, (0, 2, 3, 1))\n",
    "    \n",
    "    \n",
    "    for i in np.arange(2):\n",
    "        data = np.load('D:/7107018014/cifar10/{}/{}_data{}_batch{}.npy'.format(label_num,'test',label_num, i))\n",
    "        label = np.load('D:/7107018014/cifar10/{}/{}_label{}_batch{}.npy'.format(label_num,'test',label_num, i))\n",
    "        \n",
    "        data = np.transpose(data, (0, 2, 3, 1))\n",
    "        try:\n",
    "            x_test = np.vstack((x_test, data))\n",
    "            y_test = np.hstack((y_test, label))\n",
    "        except:\n",
    "            x_test = data\n",
    "            y_test = label\n",
    "    ## outlier data\n",
    "    for index in np.delete(np.arange(10), label_num):\n",
    "        \n",
    "        out_data = np.load('D:/7107018014/cifar10/{}/{}_data{}_batch{}.npy'.format(index,'train',index, 0))\n",
    "        out_label = np.load('D:/7107018014/cifar10/{}/{}_label{}_batch{}.npy'.format(index,'train',index, 0))\n",
    "\n",
    "        out_data = np.transpose(out_data, (0, 2, 3, 1))\n",
    "        try:\n",
    "            out_datas_ = np.vstack((out_datas_, out_data))\n",
    "            out_labels_ = np.hstack((out_labels_, out_label))\n",
    "        except:\n",
    "            out_datas_ = out_data\n",
    "            out_labels_ = out_label\n",
    "        \n",
    "    ### 製造outlier\n",
    "    random.seed (1120)\n",
    "    normal_n = len(x_train)\n",
    "    outlier_n = len(out_datas_) ####\n",
    "    indices = np.random.choice(outlier_n, int(normal_n * outlier_nu), replace=False)\n",
    "    x_train = np.vstack( (x_train[y_train == label_num], out_datas_[indices]))\n",
    "    y_train = np.hstack( (y_train[y_train == label_num], out_labels_[indices]))\n",
    "    \n",
    "    \n",
    "    # Flatten the dataset\n",
    "    x_train = x_train.reshape((-1, flatten_size))\n",
    "    x_test = x_test.reshape((-1, flatten_size))\n",
    "    \n",
    "    x_train = x_train.astype('float32')\n",
    "    x_test = x_test.astype('float32')\n",
    "    y_train = y_train.astype('float32')\n",
    "    y_test = y_test.astype('float32')\n",
    "\n",
    "    train_buf = x_train.shape[0]\n",
    "    train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
    "    train_dataset = train_dataset.shuffle(buffer_size=train_buf)\n",
    "    train_dataset = train_dataset.batch(batch_size)\n",
    "    \n",
    "    return train_dataset, x_train, x_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uwrm-Sw7Q1ir"
   },
   "source": [
    "## 建構模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7fNOIyn8Q1iv"
   },
   "source": [
    "## Prtrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Autoencoder():\n",
    "    \n",
    "    def __init__(self, label_num, img_size = img_size, z_dim = z_dim, h_dim = h_dim, img_channel = img_channel):\n",
    "        \n",
    "        self.label_num = label_num\n",
    "        self.flatten_size = img_size * img_size * img_channel\n",
    "        self.z_dim = z_dim\n",
    "        self.h_dim = h_dim\n",
    "        self.encoder = self.make_encoder()\n",
    "        self.decoder = self.make_decoder()\n",
    "        self.ae_optimizer = tf.keras.optimizers.Adam(lr = 0.001, epsilon = 1e-6)\n",
    "        \n",
    "\n",
    "    def make_encoder(self):\n",
    "\n",
    "        inputs = tf.keras.Input(shape=(self.flatten_size,), name='inputs')\n",
    "        x = tf.reshape(inputs, shape = (-1, 32, 32, 3))\n",
    "        x = tf.keras.layers.Conv2D(self.h_dim[0], (5, 5), activation='linear', padding='same',  use_bias = False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization(epsilon = 1e-04)(x)\n",
    "        x = tf.keras.layers.ELU()(x)\n",
    "        x = tf.keras.layers.MaxPool2D((2, 2), padding='same')(x)\n",
    "\n",
    "        x = tf.keras.layers.Conv2D(self.h_dim[1], (5, 5), activation='linear', padding='same',  use_bias = False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization(epsilon = 1e-04)(x)\n",
    "        x = tf.keras.layers.ELU()(x)\n",
    "        x = tf.keras.layers.MaxPool2D((2, 2), padding='same')(x)\n",
    "        \n",
    "        x = tf.keras.layers.Conv2D(self.h_dim[2], (5, 5), activation='linear', padding='same',  use_bias = False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization(epsilon = 1e-04)(x)\n",
    "        x = tf.keras.layers.ELU()(x)\n",
    "        x = tf.keras.layers.MaxPool2D((2, 2), padding='same')(x)\n",
    "\n",
    "        x = tf.keras.layers.Flatten(name = 'Flatten')(x)\n",
    "        x = tf.keras.layers.BatchNormalization(epsilon = 1e-04)(x)\n",
    "        x = tf.keras.layers.Dense(self.z_dim, use_bias = False, name = 'dense')(x)\n",
    "        \n",
    "        \n",
    "        model = tf.keras.Model(inputs=inputs, outputs=x)\n",
    "\n",
    "        return model    \n",
    "   \n",
    "    def make_decoder(self):\n",
    "\n",
    "        inputs = tf.keras.Input(shape=(self.z_dim,), name='inputs')\n",
    "        x = tf.reshape(inputs, shape = (-1, 4, 4, 8))\n",
    "        x = tf.keras.layers.LeakyReLU()(x)\n",
    "        #x = tf.keras.layers.UpSampling2D((2, 2))(x)    \n",
    "\n",
    "        x = tf.keras.layers.Conv2DTranspose(self.h_dim[2], (5, 5), padding='same',  use_bias = False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.ELU()(x)\n",
    "        x = tf.keras.layers.UpSampling2D((2, 2))(x)   \n",
    "\n",
    "        x = tf.keras.layers.Conv2DTranspose(self.h_dim[1], (5, 5), padding='same',  use_bias = False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.ELU()(x)\n",
    "        x = tf.keras.layers.UpSampling2D((2, 2))(x)\n",
    "        \n",
    "        x = tf.keras.layers.Conv2DTranspose(self.h_dim[0], (5, 5), padding='same',  use_bias = False)(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.ELU()(x)\n",
    "        x = tf.keras.layers.UpSampling2D((2, 2))(x) \n",
    "        \n",
    "        x = tf.keras.layers.Conv2DTranspose(3, (5, 5), activation='sigmoid', padding='same',  use_bias = False)(x)\n",
    "        #x = tf.reshape(x, shape = (-1, self.flatten_size))\n",
    "        x = tf.keras.layers.Flatten(name = 'Flatten')(x)\n",
    "        model = tf.keras.Model(inputs=inputs, outputs=x)\n",
    "\n",
    "        return model   \n",
    "    \n",
    "    @tf.function\n",
    "    def train_ae_model(self, batch_x):\n",
    "        with tf.GradientTape() as ae_tape:\n",
    "            ae_output = self.decoder(self.encoder(batch_x, training=True), training=True)\n",
    "            loss = tf.reduce_mean((ae_output - batch_x) ** 2)\n",
    "\n",
    "        grads = ae_tape.gradient(loss, self.encoder.trainable_variables + self.decoder.trainable_variables)\n",
    "        self.ae_optimizer.apply_gradients(zip(grads, self.encoder.trainable_variables + self.decoder.trainable_variables))\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def train_ae( self, n_ae_epochs, train_dataset, ae_lr_milestone=250 ):\n",
    "        #loss_list = []\n",
    "        for epoch in range(n_ae_epochs):\n",
    "            epoch_ae_loss_avg = tf.metrics.Mean()\n",
    "            \n",
    "            if (epoch+1) % ae_lr_milestone == 0:\n",
    "                self.ae_optimizer.lr = 0.1 * self.ae_optimizer.lr\n",
    "\n",
    "            for batch, (batch_x, batch_y) in enumerate(train_dataset):\n",
    "\n",
    "                ae_loss = self.train_ae_model(batch_x)\n",
    "                #epoch_ae_loss_avg(ae_loss)\n",
    "            #loss_list.append(epoch_ae_loss_avg.result())\n",
    "        \n",
    "        self.encoder.save_weights('D:/7107018014/checkpoint/oneclassAE_label_{}' .format(self.label_num))\n",
    "        #print('LOSS: {:.4f} '.format(epoch_ae_loss_avg.result()))\n",
    "        #plt.title('label = {}'.format(self.label_num))\n",
    "        #plt.plot(loss_list)\n",
    "        #plt.show()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ae = Autoencoder('airplane')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs (InputLayer)          [(None, 3072)]            0         \n",
      "_________________________________________________________________\n",
      "tf_op_layer_Reshape (TensorF [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 32, 32, 32)        2400      \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 32, 32, 32)        128       \n",
      "_________________________________________________________________\n",
      "elu (ELU)                    (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 16, 16, 64)        51200     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 16, 16, 64)        256       \n",
      "_________________________________________________________________\n",
      "elu_1 (ELU)                  (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 8, 8, 128)         204800    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 8, 8, 128)         512       \n",
      "_________________________________________________________________\n",
      "elu_2 (ELU)                  (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "Flatten (Flatten)            (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 2048)              8192      \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               262144    \n",
      "=================================================================\n",
      "Total params: 529,632\n",
      "Trainable params: 525,088\n",
      "Non-trainable params: 4,544\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder = ae.encoder\n",
    "encoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "inputs (InputLayer)          [(None, 128)]             0         \n",
      "_________________________________________________________________\n",
      "tf_op_layer_Reshape_1 (Tenso [(None, 4, 4, 8)]         0         \n",
      "_________________________________________________________________\n",
      "leaky_re_lu (LeakyReLU)      (None, 4, 4, 8)           0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose (Conv2DTran (None, 4, 4, 128)         25600     \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 4, 4, 128)         512       \n",
      "_________________________________________________________________\n",
      "elu_3 (ELU)                  (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "up_sampling2d (UpSampling2D) (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_1 (Conv2DTr (None, 8, 8, 64)          204800    \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 8, 8, 64)          256       \n",
      "_________________________________________________________________\n",
      "elu_4 (ELU)                  (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "up_sampling2d_1 (UpSampling2 (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_2 (Conv2DTr (None, 16, 16, 32)        51200     \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 16, 16, 32)        128       \n",
      "_________________________________________________________________\n",
      "elu_5 (ELU)                  (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "up_sampling2d_2 (UpSampling2 (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_transpose_3 (Conv2DTr (None, 32, 32, 3)         2400      \n",
      "_________________________________________________________________\n",
      "Flatten (Flatten)            (None, 3072)              0         \n",
      "=================================================================\n",
      "Total params: 284,896\n",
      "Trainable params: 284,448\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "decoder = ae.decoder\n",
    "decoder.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## result "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j-iGsWCOQ1i8"
   },
   "outputs": [],
   "source": [
    "def threshold_img(data, dists):\n",
    "    dict_score = {}\n",
    "    for i in range(len(dists)):\n",
    "        dict_score[float(dists[i])] = i\n",
    "    sort_score = [(k, dict_score[k]) for k in sorted(dict_score.keys())]\n",
    "    \n",
    "    n_digits = 20  # how many digits we will display\n",
    "    fig = plt.figure(figsize=(20, 4))\n",
    "    for i in range(1, n_digits):\n",
    "        # display normal\n",
    "        ax = plt.subplot(2, n_digits, i + 1)\n",
    "        plt.imshow(data[sort_score[i][1]].reshape(img_size, img_size, img_channel), cmap = plt.cm.gray)\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "        # display anomaly\n",
    "        ax = plt.subplot(2, n_digits, i + 1 + n_digits)\n",
    "        plt.imshow(data[sort_score[-i][1]].reshape(img_size, img_size, img_channel), cmap = plt.cm.gray)\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def threshold_uncertain_img(data, dists, r):\n",
    "    dist2radius = (dists - r**2)\n",
    "    normal = dist2radius[dist2radius<0]\n",
    "    anomaly = dist2radius[dist2radius>0]\n",
    "    normal_score = {}\n",
    "    anomaly_score = {}\n",
    "    for i in range(len(normal)):\n",
    "        normal_score[float(normal[i])] = i\n",
    "    for i in range(len(anomaly)):\n",
    "        anomaly_score[float(anomaly[i])] = i\n",
    "    normalsort_score = [(k, normal_score[k]) for k in sorted(normal_score.keys())]\n",
    "    anomalysort_score = [(k, anomaly_score[k]) for k in sorted(anomaly_score.keys())]\n",
    "    \n",
    "    n_digits = min([len(normal),len(anomaly),20])#20  # how many digits we will display\n",
    "    fig = plt.figure(figsize=(20, 4))\n",
    "    #print('normal:', len(normal), 'anomaly', len(anomaly))\n",
    "    for i in range(1, n_digits):\n",
    "        # display normal\n",
    "        \n",
    "        ax = plt.subplot(2, n_digits, i + 1)\n",
    "        plt.imshow(data[normalsort_score[-i][1]].reshape(img_size, img_size, img_channel), cmap = plt.cm.gray)\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "        # display anomaly\n",
    "        ax = plt.subplot(2, n_digits, i + 1 + n_digits)\n",
    "        plt.imshow(data[anomalysort_score[i][1]].reshape(img_size, img_size, img_channel), cmap = plt.cm.gray)\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UKX2VlDnQ1i5"
   },
   "outputs": [],
   "source": [
    "def distance_plot(label_num, dist, data, radius_end, objective, train = True, label = None,  nu = 0.1):\n",
    "    treshold = radius_end ** 2\n",
    "    if train:\n",
    "        plt.title(str(objective) + 'label = {} train data'.format(label_num))\n",
    "        if objective == 'soft-boundary':\n",
    "            treshold = 0\n",
    "            plt.hist(dis, rwidth=0.3, color='b' ,bins = 100)\n",
    "            plt.axvline(treshold,  color='r')\n",
    "        else:\n",
    "            score = dist\n",
    "            plt.hist(score, rwidth=0.3, color='b' ,bins = 100)\n",
    "            plt.axvline(float(treshold),  color='g')\n",
    "        plt.show()       \n",
    "    \n",
    "    else:\n",
    "        plt.title(str(objective) + 'label = {} test data'.format(label_num))\n",
    "        if objective == 'soft-boundary':\n",
    "            plt.hist(dis[label!=label_num], rwidth=0.3, color='r' ,bins = 100,label = 'anomaly')\n",
    "            plt.hist(dis[label==label_num], rwidth = 0.3, color = 'b',label = 'normal', bins = 100)\n",
    "            plt.axvline(treshold,  color='g')\n",
    "            plt.legend()\n",
    "            plt.show()\n",
    "            plt.hist(dist[label==label_num], rwidth=0.3, color='b' ,bins = 100,label = 'normal')\n",
    "            plt.axvline(treshold, color='g')\n",
    "            plt.legend()\n",
    "            plt.show()\n",
    "            plt.hist(dist[label!=label_num], rwidth=0.3, color='r' ,bins = 100,label = 'anomaly')\n",
    "            plt.axvline(treshold, color='g')\n",
    "        else:\n",
    "            score = dist\n",
    "            plt.hist(score[label!=label_num], rwidth=0.3, color='r' ,bins = 100,label = 'anomaly')\n",
    "            plt.hist(score[label==label_num], rwidth=0.3, color='b' ,bins = 100,label = 'normal')\n",
    "            plt.axvline(float(treshold), color='g')\n",
    "            plt.legend()\n",
    "            plt.show()\n",
    "            plt.hist(score[label==label_num], rwidth=0.3, color='b' ,bins = 100,label = 'normal')\n",
    "            plt.axvline(float(treshold), color='g')\n",
    "            plt.legend()\n",
    "            plt.show()\n",
    "            plt.hist(score[label!=label_num], rwidth=0.3, color='r' ,bins = 100,label = 'anomaly')\n",
    "            plt.axvline(float(treshold), color='g')\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def false_img(data, labels_,  dists, r, label_num):\n",
    "    dist2radius = (dists - r**2)\n",
    "    dist_n = dist2radius[labels_ != label_num]\n",
    "    dist_p = dist2radius[labels_ == label_num]\n",
    "    data_n = data[labels_ != label_num]\n",
    "    data_p = data[labels_ == label_num]\n",
    "    fn = dist_n[dist_n<0]\n",
    "    fp = dist_p[dist_p>0]\n",
    "    fn_score = {}\n",
    "    fp_score = {}\n",
    "    for i in range(len(fn)):\n",
    "        fn_score[float(fn[i])] = i\n",
    "    for i in range(len(fp)):\n",
    "        fp_score[float(fp[i])] = i\n",
    "        \n",
    "    fn_score_sort = [(k, fn_score[k]) for k in sorted(fn_score.keys())]\n",
    "    fp_score_sort = [(k, fp_score[k]) for k in sorted(fp_score.keys())]\n",
    "    \n",
    "    n_digits = min([len(fn),len(fp),20])#20  # how many digits we will display\n",
    "    fig = plt.figure(figsize=(20, 4))\n",
    "    #print('normal:', len(normal), 'anomaly', len(anomaly))\n",
    "    for i in range(1, n_digits):\n",
    "        # display fn\n",
    "        ax = plt.subplot(2, n_digits, i + 1)\n",
    "        plt.imshow(data_n[fn_score_sort[-i][1]].reshape(img_size, img_size, img_channel), cmap = plt.cm.gray)\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "        # display fp\n",
    "        ax = plt.subplot(2, n_digits, i + 1 + n_digits)\n",
    "        plt.imshow(data_p[fp_score_sort[i][1]].reshape(img_size, img_size, img_channel), cmap = plt.cm.gray)\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class make_binary_model():\n",
    "    def __init__(self, encoder, z_dim = z_dim):\n",
    "        self.encoder = encoder\n",
    "        self.z_dim = z_dim\n",
    "        self.SVDD_model = self.make_model()\n",
    "    \n",
    "    \n",
    "    def initializer(self):\n",
    "        kernel_initializer = self.encoder.get_layer('dense').get_weights()[0]\n",
    "        \n",
    "        return kernel_initializer\n",
    "    \n",
    "    def make_model(self):\n",
    "        \n",
    "        kernel_initializer = self.initializer()\n",
    "        \n",
    "        self.encoder.trainable = True\n",
    "        x = self.encoder.get_layer('Flatten').output\n",
    "        x = tf.keras.layers.Dense(\n",
    "        self.z_dim, activation = 'relu', kernel_initializer = tf.constant_initializer(kernel_initializer))(x)\n",
    "        outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n",
    "        model_new = tf.keras.Model(inputs = self.encoder.input, outputs = outputs)\n",
    "    \n",
    "        return model_new\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GbTlov0uQ1it"
   },
   "outputs": [],
   "source": [
    "class train_classfication():\n",
    "    \n",
    "    def __init__(self, label_num , binary_model, train_dataset, nu ):\n",
    "        names = np.array(['airplane',\n",
    "        'automobile',\n",
    "        'bird',\n",
    "        'cat',\n",
    "        'deer',\n",
    "        'dog',\n",
    "        'frog',\n",
    "        'horse',\n",
    "        'ship',\n",
    "        'truck'])\n",
    "        \n",
    "        self.label_num = label_num\n",
    "        self.label_name = names[label_num]\n",
    "        self.nu = nu\n",
    "        self.binary_model = binary_model\n",
    "        self.label_num = label_num\n",
    "        self.optimizer = tf.keras.optimizers.Adam(lr = 0.0001, epsilon = 1e-6)\n",
    "        self.train_dataset = train_dataset\n",
    "        \n",
    "        \n",
    "    def loss(self, batch_x, batch_y):\n",
    "        \n",
    "        avg_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=batch_y, logits=batch_x)\n",
    "        end_loss = tf.reduce_mean(avg_loss)\n",
    "\n",
    "        return end_loss\n",
    "\n",
    "    @tf.function\n",
    "    def train_model(self, batch_x, batch_y):\n",
    "        with tf.GradientTape() as tape:\n",
    "            output = self.binary_model(batch_x, training=True)\n",
    "            output = tf.squeeze(output)\n",
    "            output_loss = self.loss(output, batch_y)\n",
    "\n",
    "        grads = tape.gradient(output_loss, self.binary_model.trainable_variables)\n",
    "        self.optimizer.apply_gradients(zip(grads, self.binary_model.trainable_variables))\n",
    "\n",
    "        return output_loss\n",
    "    \n",
    "    def train( self, lr_milestone = 15):\n",
    "        \n",
    "        \n",
    "        pre_loss = np.inf\n",
    "        loss = 0.\n",
    "        loss_list = []\n",
    "        for epoch in range(60):\n",
    "            epoch_ae_loss_avg = tf.metrics.Mean()\n",
    "            \n",
    "            if (epoch+1) % lr_milestone == 0:\n",
    "                self.optimizer.lr = 0.1 * self.optimizer.lr\n",
    "\n",
    "            for batch, (batch_x, batch_y) in enumerate(train_dataset):\n",
    "\n",
    "                ae_loss = self.train_model(batch_x, batch_y)\n",
    "                epoch_ae_loss_avg(ae_loss)\n",
    "            loss_list.append(epoch_ae_loss_avg.result())\n",
    "        \n",
    "        self.binary_model.save_weights('D:/7107018014/checkpoint/binary_model_label_{}_outlier_{}' .format(\n",
    "            self.label_name, self.nu))\n",
    "        \n",
    "        \n",
    "    def score(self, data, label):\n",
    "        self.binary_model.load_weights('D:/7107018014/checkpoint/binary_model_label_{}_outlier_{}' .format(\n",
    "            self.label_name, self.nu))\n",
    "        batch_size = 200\n",
    "        train_dataset = tf.data.Dataset.from_tensor_slices((data, label))\n",
    "        train_dataset = train_dataset.batch(batch_size)\n",
    "\n",
    "        for batch, (batch_x,batch_y) in enumerate(train_dataset):\n",
    "            output = self.binary_model(batch_x, training=False)\n",
    "            \n",
    "            if batch ==0:\n",
    "                modeloutput = output\n",
    "                labels_ = batch_y\n",
    "                datas = batch_x\n",
    "            else:\n",
    "                modeloutput = tf.concat([modeloutput, output], 0)\n",
    "                labels_ = tf.concat([labels_, batch_y], 0)\n",
    "                datas = tf.concat([datas, batch_x], 0)\n",
    "                \n",
    "        labels_ = labels_.numpy()\n",
    "        datas = datas.numpy()\n",
    "        \n",
    "        return modeloutput, labels_, datas\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    def result(self, scores, y_test, show_img = False):\n",
    "        \n",
    "        y_true = y_test==self.label_num\n",
    "        test_auc = roc_auc_score(y_true, scores)\n",
    "        if show_img:\n",
    "            print('AUC:', test_auc)\n",
    "            \n",
    "        return test_auc \n",
    "    \n",
    "    def test_result(self, x_test, y_test, show_img = False, load_model = False, search_u = False):\n",
    "        \n",
    "        scores, labels_, datas = self.score(x_test, y_test)\n",
    "        test_auc = self.result(scores, labels_, show_img = show_img)\n",
    "        if show_img:\n",
    "            distance_plot(self.label_num, dist, self.objective, name =  self.label_num,\n",
    "                          train = False, label = y_test, nu = self.nu )\n",
    "            print('test label = {} uncertain_img'.format(self.label_num))\n",
    "            threshold_uncertain_img(x_test, dist, radius_end)\n",
    "            print('test label = {} distance oder'.format(self.label_num))\n",
    "            threshold_img(x_test, dist)\n",
    "            \n",
    "        if search_u:\n",
    "            print('u = {}, auc = {}'.format(self.nu, test_auc))\n",
    "        return test_auc\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading  airplane\n",
      "auc =  0.5116491111111111\n",
      "Loading  automobile\n",
      "auc =  0.5624785555555556\n",
      "Loading  bird\n",
      "auc =  0.5000555555555555\n",
      "Loading  cat\n",
      "auc =  0.5\n",
      "Loading  deer\n",
      "auc =  0.47885738888888896\n",
      "Loading  dog\n",
      "auc =  0.5003335555555556\n",
      "Loading  frog\n",
      "auc =  0.5028213333333332\n",
      "Loading  horse\n",
      "auc =  0.48058877777777775\n",
      "Loading  ship\n",
      "auc =  0.5182690555555555\n",
      "Loading  truck\n",
      "auc =  0.5126814444444444\n"
     ]
    }
   ],
   "source": [
    "nu = 0.0\n",
    "outlier_nu = 0.\n",
    "lr_milestone = 50\n",
    "result = np.zeros((10, 4))\n",
    "\n",
    "for label_num in np.arange(10):\n",
    "    print(\"Loading \",names[label_num])\n",
    "    train_dataset, x_train, x_test, y_train, y_test = load_data(label_num = label_num, outlier_nu = outlier_nu)\n",
    "    pre_time = time.time()\n",
    "    ae = Autoencoder(label_num)\n",
    "    ae.train_ae( 50, train_dataset)\n",
    "    encoder = ae.encoder\n",
    "    make_binary = make_binary_model(encoder)\n",
    "    binary_model = make_binary.make_model() \n",
    "    classfication = train_classfication(label_num , binary_model, train_dataset, nu )\n",
    "    classfication.train(lr_milestone)\n",
    "    test_auc = classfication.test_result(x_test, y_test, load_model = True)\n",
    "    train_time = time.time() - pre_time\n",
    "    print('auc = ', test_auc)\n",
    "    result[label_num, 0] = label_num\n",
    "    result[label_num, 2] = test_auc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.51164911, 0.        ],\n",
       "       [1.        , 0.        , 0.56247856, 0.        ],\n",
       "       [2.        , 0.        , 0.50005556, 0.        ],\n",
       "       [3.        , 0.        , 0.5       , 0.        ],\n",
       "       [4.        , 0.        , 0.47885739, 0.        ],\n",
       "       [5.        , 0.        , 0.50033356, 0.        ],\n",
       "       [6.        , 0.        , 0.50282133, 0.        ],\n",
       "       [7.        , 0.        , 0.48058878, 0.        ],\n",
       "       [8.        , 0.        , 0.51826906, 0.        ],\n",
       "       [9.        , 0.        , 0.51268144, 0.        ]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading  airplane\n",
      "auc =  0.5802643888888889\n",
      "Loading  automobile\n",
      "auc =  0.499404\n",
      "Loading  bird\n",
      "auc =  0.5\n",
      "Loading  cat\n",
      "auc =  0.5000555555555555\n",
      "Loading  deer\n",
      "auc =  0.4985548333333333\n",
      "Loading  dog\n",
      "auc =  0.5\n",
      "Loading  frog\n",
      "auc =  0.4146121666666666\n",
      "Loading  horse\n",
      "auc =  0.5\n",
      "Loading  ship\n",
      "auc =  0.49727655555555555\n",
      "Loading  truck\n",
      "auc =  0.500278\n"
     ]
    }
   ],
   "source": [
    "nu = 0.1\n",
    "outlier_nu = 0.1\n",
    "lr_milestone = 50\n",
    "result = np.zeros((10, 4))\n",
    "\n",
    "for label_num in np.arange(10):\n",
    "    print(\"Loading \",names[label_num])\n",
    "    train_dataset, x_train, x_test, y_train, y_test = load_data(label_num = label_num, outlier_nu = outlier_nu)\n",
    "    pre_time = time.time()\n",
    "    ae = Autoencoder(label_num)\n",
    "    ae.train_ae( 350, train_dataset)\n",
    "    encoder = ae.encoder\n",
    "    make_binary = make_binary_model(encoder)\n",
    "    binary_model = make_binary.make_model() \n",
    "    classfication = train_classfication(label_num , binary_model, train_dataset, nu )\n",
    "    classfication.train(lr_milestone)\n",
    "    test_auc = classfication.test_result(x_test, y_test, load_model = True)\n",
    "    train_time = time.time() - pre_time\n",
    "    print('auc = ', test_auc)\n",
    "    result[label_num, 0] = label_num\n",
    "    result[label_num, 2] = test_auc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading  airplane\n",
      "auc =  0.4030138333333334\n",
      "Loading  automobile\n",
      "auc =  0.5010073888888888\n",
      "Loading  bird\n",
      "auc =  0.4864466666666667\n",
      "Loading  cat\n",
      "auc =  0.509422111111111\n",
      "Loading  deer\n",
      "auc =  0.5\n",
      "Loading  dog\n",
      "auc =  0.5\n",
      "Loading  frog\n",
      "auc =  0.49997394444444443\n",
      "Loading  horse\n",
      "auc =  0.5052738888888889\n",
      "Loading  ship\n",
      "auc =  0.4962263333333333\n",
      "Loading  truck\n",
      "auc =  0.49036027777777774\n"
     ]
    }
   ],
   "source": [
    "nu = 0.2\n",
    "outlier_nu = 0.2\n",
    "lr_milestone = 50\n",
    "result = np.zeros((10, 4))\n",
    "\n",
    "for label_num in np.arange(10):\n",
    "    print(\"Loading \",names[label_num])\n",
    "    train_dataset, x_train, x_test, y_train, y_test = load_data(label_num = label_num, outlier_nu = outlier_nu)\n",
    "    pre_time = time.time()\n",
    "    ae = Autoencoder(label_num)\n",
    "    ae.train_ae( 350, train_dataset)\n",
    "    encoder = ae.encoder\n",
    "    make_binary = make_binary_model(encoder)\n",
    "    binary_model = make_binary.make_model() \n",
    "    classfication = train_classfication(label_num , binary_model, train_dataset, nu )\n",
    "    classfication.train(lr_milestone)\n",
    "    test_auc = classfication.test_result(x_test, y_test, load_model = True)\n",
    "    train_time = time.time() - pre_time\n",
    "    print('auc = ', test_auc)\n",
    "    result[label_num, 0] = label_num\n",
    "    result[label_num, 2] = test_auc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading  airplane\n",
      "auc =  0.47572288888888886\n",
      "Loading  automobile\n",
      "auc =  0.5058556666666667\n",
      "Loading  bird\n",
      "auc =  0.5\n",
      "Loading  cat\n",
      "auc =  0.500391\n",
      "Loading  deer\n",
      "auc =  0.5\n",
      "Loading  dog\n",
      "auc =  0.5\n",
      "Loading  frog\n",
      "auc =  0.5367641111111112\n",
      "Loading  horse\n",
      "auc =  0.5006681666666667\n",
      "Loading  ship\n",
      "auc =  0.46861366666666665\n",
      "Loading  truck\n",
      "auc =  0.6304242222222222\n"
     ]
    }
   ],
   "source": [
    "nu = 0.3\n",
    "outlier_nu = 0.3\n",
    "lr_milestone = 50\n",
    "result = np.zeros((10, 4))\n",
    "\n",
    "for label_num in np.arange(10):\n",
    "    print(\"Loading \",names[label_num])\n",
    "    train_dataset, x_train, x_test, y_train, y_test = load_data(label_num = label_num, outlier_nu = outlier_nu)\n",
    "    pre_time = time.time()\n",
    "    ae = Autoencoder(label_num)\n",
    "    ae.train_ae( 350, train_dataset)\n",
    "    encoder = ae.encoder\n",
    "    make_binary = make_binary_model(encoder)\n",
    "    binary_model = make_binary.make_model() \n",
    "    classfication = train_classfication(label_num , binary_model, train_dataset, nu )\n",
    "    classfication.train(lr_milestone)\n",
    "    test_auc = classfication.test_result(x_test, y_test, load_model = True)\n",
    "    train_time = time.time() - pre_time\n",
    "    print('auc = ', test_auc)\n",
    "    result[label_num, 0] = label_num\n",
    "    result[label_num, 2] = test_auc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "SVDD_ver2_test.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}