train_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import library\n",
    "\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "import keras\n",
    "from keras.layers import (Input, Activation, Conv3D, Dense, Dropout, Flatten, \n",
    "                          MaxPooling3D, BatchNormalization, AveragePooling3D, \n",
    "                          Reshape, Lambda, GlobalAveragePooling3D, Concatenate,\n",
    "                          ReLU, Add)\n",
    "\n",
    "from keras.losses import categorical_crossentropy\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.optimizers import gradient_descent_v2\n",
    "from keras.utils import np_utils\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.utils import layer_utils\n",
    "from keras.utils.layer_utils import get_source_inputs\n",
    "from keras.utils.data_utils import get_file\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.backend import set_session\n",
    "\n",
    "from keras.callbacks import CSVLogger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_history(history, result_dir):\n",
    "    '''\n",
    "    Plots the accuracy and loss graphs of train and val and saves them.\n",
    "    '''\n",
    "\n",
    "    plt.plot(history.history['accuracy'], marker='.')\n",
    "    plt.plot(history.history['val_accuracy'], marker='.')\n",
    "    plt.title('model accuracy')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('accuracy')\n",
    "    plt.grid()\n",
    "    plt.legend(['accuracy', 'val_accuracy'], loc='lower right')\n",
    "    plt.savefig(os.path.join(result_dir, 'model_accuracy.png'))\n",
    "    plt.show()\n",
    "\n",
    "    plt.plot(history.history['loss'], marker='.')\n",
    "    plt.plot(history.history['val_loss'], marker='.')\n",
    "    plt.title('model loss')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('loss')\n",
    "    plt.grid()\n",
    "    plt.legend(['loss', 'val_loss'], loc='upper right')\n",
    "    plt.savefig(os.path.join(result_dir, 'model_loss.png'))\n",
    "    plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Videoto3D:\n",
    "\n",
    "    def __init__(self, width, height, depth):\n",
    "        self.width = width\n",
    "        self.height = height\n",
    "        self.depth = depth\n",
    "\n",
    "    def video3d(self, filename):\n",
    "        \n",
    "        frames = []\n",
    "        index = len(os.listdir(filename)) // self.depth\n",
    "        images = os.listdir(filename)[::index]\n",
    "        images = images[0:25]\n",
    "        images.sort()\n",
    "\n",
    "        for img in images:\n",
    "\n",
    "            img_path = os.path.join(filename, img)\n",
    "            frame = cv2.imread(img_path)\n",
    "            frame = cv2.resize(frame, (self.height, self.width))\n",
    "            frames.append(frame)\n",
    "\n",
    "        return np.array(frames) / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(video_dir, result_dir, nb_classes = 101, img_size = 224, frames = 25):\n",
    "    '''\n",
    "    Preprocess the videos into X and Y and saves in npz format and \n",
    "    computes input shape\n",
    "    '''\n",
    "\n",
    "    img_rows, img_cols  = img_size, img_size\n",
    "\n",
    "    channel = 3\n",
    "\n",
    "    files = os.listdir(video_dir)\n",
    "    files.sort()\n",
    "\n",
    "    if '.ipynb_checkpoints' in files:\n",
    "        files.remove('.ipynb_checkpoints')\n",
    "\n",
    "    X = []\n",
    "    labels = []\n",
    "    labellist = []\n",
    "\n",
    "    # Obtain labels and X\n",
    "    for filename in files:\n",
    "\n",
    "        name = os.path.join(video_dir, filename)\n",
    "        \n",
    "        for v_files in os.listdir(name):\n",
    "            \n",
    "            v_file_path = os.path.join(name, v_files)\n",
    "            label = filename\n",
    "            if label not in labellist:\n",
    "                if len(labellist) >= nb_classes:\n",
    "                    continue\n",
    "                labellist.append(label)\n",
    "            labels.append(label)\n",
    "            X.append(v_file_path)\n",
    "\n",
    "    if not os.path.isdir(result_dir):\n",
    "        os.makedir(result_dir)\n",
    "    with open(os.path.join(result_dir, 'classes.txt'), 'w') as fp:\n",
    "        for i in range(len(labellist)):\n",
    "            fp.write('{} {}\\n'.format(i, labellist[i]))\n",
    "\n",
    "    for num, label in enumerate(labellist):\n",
    "        for i in range(len(labels)):\n",
    "            if label == labels[i]:\n",
    "                labels[i] = num\n",
    "                \n",
    "    Y = np_utils.to_categorical(labels, nb_classes)\n",
    "\n",
    "    print('X_shape:{}\\tY_shape:{}'.format(len(X), Y.shape))\n",
    "\n",
    "    input_shape = (frames, img_rows, img_cols, channel)\n",
    "\n",
    "    return X, Y, input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.slowfast import *\n",
    "\n",
    "def resnet50(inputs, **kwargs):\n",
    "    model = SlowFast_body(inputs, [3, 4, 6, 3], bottleneck, **kwargs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.i3dinception import *\n",
    "\n",
    "def I3DModel(model, nb_classes):\n",
    "\n",
    "    x = model.layers[-1].output\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = conv3d_bn(x, nb_classes, 1, 1, 1, padding='same', \n",
    "            use_bias=True, use_activation_fn=False, use_bn=False, name='Conv3d_6a_1x1')\n",
    "    num_frames_remaining = int(x.shape[1])\n",
    "    x = Reshape((num_frames_remaining, nb_classes))(x)\n",
    "    # logits (raw scores for each class)\n",
    "    x = Lambda(lambda x: K.mean(x, axis=1, keepdims=False),\n",
    "                output_shape=lambda s: (s[0], s[2]))(x)\n",
    "    x = Activation('softmax', name='prediction')(x)\n",
    "    model = Model(model.inputs, x)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class batchGenerator(keras.utils.all_utils.Sequence):\n",
    "\n",
    "    def __init__(self, x_set, y_set, batch_size, vid3d):\n",
    "        self.x, self.y = x_set, y_set\n",
    "        self.batch_size = batch_size\n",
    "        self.vid3d = vid3d\n",
    "\n",
    "    def __len__(self):\n",
    "        return math.ceil(len(self.x) / self.batch_size)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        batch_x = []\n",
    "\n",
    "        for video in self.x[idx * self.batch_size:(idx + 1) * self.batch_size]:\n",
    "            batch_x.append(self.vid3d.video3d(video))\n",
    "\n",
    "        batch_x = np.array(batch_x)\n",
    "        batch_x = batch_x.astype('float32')\n",
    "        batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size]\n",
    "\n",
    "        return batch_x, batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(video_dir, result_dir, nb_classes = 101, batch_size = 32, epochs = 100, img_size = 224, frames = 25):\n",
    "\n",
    "    X, Y, input_shape = preprocess(video_dir, result_dir, nb_classes, img_size, \n",
    "                                   frames)\n",
    "\n",
    "    print(\"Input Shape = \", input_shape)\n",
    "\n",
    "    vid3d = Videoto3D(img_size, img_size, frames)\n",
    "\n",
    "    ## For i3D Inception model ##\n",
    "#     i3dmodel = Inception_Inflated3d(include_top=False,\n",
    "#                 weights='rgb_imagenet_and_kinetics',\n",
    "#                 input_tensor=None,\n",
    "#                 input_shape=input_shape,\n",
    "#                 dropout_prob=0.5,\n",
    "#                 endpoint_logit=False)\n",
    "    \n",
    "#     model = I3DModel(i3dmodel, nb_classes)\n",
    "    \n",
    "    ## For Slowfast model ##\n",
    "    x = Input(shape = input_shape)\n",
    "    model = resnet50(x, num_classes=nb_classes)\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=gradient_descent_v2.SGD(learning_rate=0.01, momentum=0.9), \n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "#     model = load_model(os.path.join(result_dir, \"slowfast-77-0.73.hd5\"))\n",
    "    \n",
    "    model.summary()\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(\n",
    "        X, Y, test_size=0.15, shuffle = True)\n",
    "    \n",
    "    X_train, X_val, Y_train, Y_val = train_test_split(\n",
    "        X_train, Y_train, test_size=0.15, shuffle = True)\n",
    "\n",
    "    print(\"X_train.shape = \", len(X_train))\n",
    "    print(\"X_val.shape = \", len(X_val))\n",
    "    print(\"X_test.shape = \", len(X_test))\n",
    "\n",
    "    # MODEL CHECK POINTS #\n",
    "\n",
    "    csv_logger = CSVLogger(os.path.join(result_dir, \"slowfast_model_history_log.csv\"), append=True)\n",
    "\n",
    "    filepath = os.path.join(result_dir, \"slowfast-{epoch:02d}-{val_accuracy:.2f}.hdf5\")\n",
    "    checkpoint = keras.callbacks.ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, mode='max')\n",
    "    callbacks_list = [checkpoint, csv_logger]\n",
    "\n",
    "    history = model.fit(batchGenerator(X_train, Y_train, batch_size, vid3d), steps_per_epoch = math.ceil(len(X_train) / batch_size), \n",
    "                                  validation_data = batchGenerator(X_val, Y_val, batch_size, vid3d), validation_steps = math.ceil(len(X_val) / batch_size), \n",
    "                                  epochs = epochs, verbose = 1, callbacks=callbacks_list, initial_epoch = 0)\n",
    "\n",
    "    model_json = model.to_json()\n",
    "    \n",
    "    if not os.path.isdir(result_dir):\n",
    "        os.makedir(result_dir)\n",
    "    with open(os.path.join(result_dir, 'slowfast.json'), 'w') as json_file:\n",
    "        json_file.write(model_json)\n",
    "    \n",
    "    model.save_weights(os.path.join(result_dir, 'slowfast_finalweights.hdf5'))\n",
    "\n",
    "    model.save(os.path.join(result_dir, \"slowfast_finalmodel.hdf5\"))\n",
    "\n",
    "    loss, acc = model.evaluate(batchGenerator(X_test, Y_test, batch_size, vid3d),\n",
    "                               steps = math.ceil(len(X_test) / batch_size), verbose = 1)\n",
    "    \n",
    "    plot_history(history, result_dir)\n",
    "\n",
    "    print('Test loss:', loss)\n",
    "    print('Test accuracy:', acc)\n",
    "\n",
    "    return history, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TRAIN MODEL ##\n",
    "\n",
    "history, model = main(video_dir = 'DCSASS Dataset/', result_dir = 'output/', nb_classes = 14, batch_size = 8, epochs = 100, img_size = 224, frames = 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def frames_from_video(video_dir, nb_frames = 25, img_size = 224):\n",
    "\n",
    "    # Opens the Video file\n",
    "    cap = cv2.VideoCapture(video_dir)\n",
    "    i=0\n",
    "    frames = []\n",
    "    while(cap.isOpened() and i<nb_frames):\n",
    "        ret, frame = cap.read()\n",
    "        if ret == False:\n",
    "            break\n",
    "        frame = cv2.resize(frame, (img_size, img_size))\n",
    "        frames.append(frame)\n",
    "        i+=1\n",
    "\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "    return np.array(frames) / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions(video_dir, model, nb_frames = 25, img_size = 224):\n",
    "\n",
    "    X = frames_from_video(video_dir, nb_frames, img_size)\n",
    "    X = np.reshape(X, (1, nb_frames, img_size, img_size, 3))\n",
    "    \n",
    "    predictions = model.predict(X)\n",
    "    preds = predictions.argmax(axis = 1)\n",
    "\n",
    "    classes = []\n",
    "    with open(os.path.join('output', 'classes.txt'), 'r') as fp:\n",
    "        for line in fp:\n",
    "            classes.append(line.split()[1])\n",
    "\n",
    "    for i in range(len(preds)):\n",
    "        print('Prediction - {} -- {}'.format(preds[i], classes[preds[i]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## LOAD MODEL ##\n",
    "\n",
    "model = load_model('output/slowfast_finalmodel.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## MAKE PREDICTIONS ##\n",
    "\n",
    "predictions(video_dir = 'test/Arrest048_x264_21.mp4', model = model, nb_frames = 25, img_size = 224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}