Vanishing_Gradient_Normalize_ver_2.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.14",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "gpu",
      "dataSources": [],
      "dockerImageVersionId": 30786,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    },
    "colab": {
      "name": "Vanishing_Gradient_Normalize_ver_2",
      "provenance": []
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the read-only \"../input/\" directory\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
        "\n",
        "import os\n",
        "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirname, filename))\n",
        "\n",
        "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n",
        "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
      ],
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true,
        "id": "Ehj_n2BhvypR"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torch import nn\n",
        "from torch.utils.data import Dataset, DataLoader, random_split\n",
        "from torchvision.datasets import FashionMNIST\n",
        "import os\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-11-30T16:55:29.649937Z",
          "iopub.execute_input": "2024-11-30T16:55:29.650311Z",
          "iopub.status.idle": "2024-11-30T16:55:31.141734Z",
          "shell.execute_reply.started": "2024-11-30T16:55:29.650277Z",
          "shell.execute_reply": "2024-11-30T16:55:31.140889Z"
        },
        "id": "AzhbkwAMvypS"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "def set_seed(seed):\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.cuda.manual_seed_all(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "\n",
        "SEED = 42\n",
        "set_seed(SEED)"
      ],
      "metadata": {
        "trusted": true,
        "id": "tmZ6WgI5vypT"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "root_dir = '/kaggle/working/'\n",
        "train_dataset = FashionMNIST(os.path.join(root_dir, './data'), train= True, download = True, transform = transforms.ToTensor())\n",
        "test_dataset = FashionMNIST(os.path.join(root_dir, './data'), train= False, download = True, transform = transforms.ToTensor())"
      ],
      "metadata": {
        "trusted": true,
        "id": "lXCT0FLyvypT"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 512\n",
        "train_ratio = 0.9\n",
        "\n",
        "train_size = int(len(train_dataset) * 0.9)\n",
        "val_size = len(train_dataset) - train_size\n",
        "\n",
        "train_subset, val_subset = random_split(train_dataset,[train_size, val_size])\n",
        "\n",
        "train_loader = DataLoader(train_subset, batch_size = batch_size, shuffle=True)\n",
        "val_loader = DataLoader(val_subset, batch_size = batch_size, shuffle=False)\n",
        "test_loader = DataLoader(test_dataset, batch_size = batch_size, shuffle=False)\n",
        "\n",
        "print(f\"Train size: {len(train_subset)}\")\n",
        "print(f\"Validation size: {len(val_subset)}\")\n",
        "print(f\"Test size: {len(test_dataset)}\")"
      ],
      "metadata": {
        "trusted": true,
        "id": "Qjcr15VRvypT"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "class MyNormalization(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "\n",
        "    def forward(self, x):\n",
        "        mean = torch.mean(x)\n",
        "        std = torch.std(x)\n",
        "\n",
        "        return (x - mean) / std\n",
        "\n",
        "x = torch.tensor([[1.0, 2.0, 3.0],\n",
        "                  [4.0, 5.0, 6.0]])\n",
        "# mean = 3.5\n",
        "# std = 1.8708\n",
        "print(MyNormalization()(x))"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-11-30T16:57:44.547219Z",
          "iopub.execute_input": "2024-11-30T16:57:44.547633Z",
          "iopub.status.idle": "2024-11-30T16:57:44.557459Z",
          "shell.execute_reply.started": "2024-11-30T16:57:44.54759Z",
          "shell.execute_reply": "2024-11-30T16:57:44.556498Z"
        },
        "id": "231M8tuuvypT"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "class MLP(nn.Module):\n",
        "    def __init__(self, input_dims, hidden_dims, output_dims):\n",
        "        super(MLP, self).__init__()\n",
        "        self.hidden_dims = hidden_dims\n",
        "\n",
        "        self.layer1 = nn.Linear(input_dims, hidden_dims)\n",
        "        self.bn1 = MyNormalization()\n",
        "\n",
        "        self.layer2 = nn.Linear(hidden_dims, hidden_dims)\n",
        "        self.bn2 = MyNormalization()\n",
        "\n",
        "        self.layer3 = nn.Linear(hidden_dims, hidden_dims)\n",
        "        self.bn3 = MyNormalization()\n",
        "\n",
        "        self.layer4 = nn.Linear(hidden_dims, hidden_dims)\n",
        "        self.bn4 = MyNormalization()\n",
        "\n",
        "        self.layer5 = nn.Linear(hidden_dims, hidden_dims)\n",
        "        self.bn5 = MyNormalization()\n",
        "\n",
        "        self.layer6 = nn.Linear(hidden_dims, hidden_dims)\n",
        "        self.bn6 = MyNormalization()\n",
        "\n",
        "        self.layer7 = nn.Linear(hidden_dims, hidden_dims)\n",
        "        self.bn7 = MyNormalization()\n",
        "\n",
        "        self.output = nn.Linear(hidden_dims, output_dims)\n",
        "\n",
        "        for module in self.modules():\n",
        "            if isinstance(module, nn.Linear):\n",
        "                nn.init.normal_(module.weight, mean=0.0, std =0.05)\n",
        "                nn.init.constant_(module.bias, 0)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = nn.Flatten()(x)\n",
        "        x = self.layer1(x)\n",
        "        x = self.bn1(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer2(x)\n",
        "        x = self.bn2(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer3(x)\n",
        "        x = self.bn3(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer4(x)\n",
        "        x = self.bn4(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer5(x)\n",
        "        x = self.bn5(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer6(x)\n",
        "        x = self.bn6(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        x = self.layer7(x)\n",
        "        x = self.bn7(x)\n",
        "        x = nn.Sigmoid()(x)\n",
        "\n",
        "        out = self.output(x)\n",
        "        return out\n",
        "\n",
        "def compute_accuracy(preds, labels):\n",
        "    acc = (torch.argmax(preds, dim = 1) == labels).sum()\n",
        "    return acc"
      ],
      "metadata": {
        "trusted": true,
        "id": "58LsHbSrvypU"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "input_dims = 784\n",
        "hidden_dims = 128\n",
        "output_dims = 10\n",
        "lr = 1e-2\n",
        "\n",
        "model = MLP(input_dims=input_dims, hidden_dims=hidden_dims, output_dims=output_dims).to(device)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr)"
      ],
      "metadata": {
        "trusted": true,
        "id": "oY9mpozhvypU"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 100\n",
        "train_loss_lst = []\n",
        "train_acc_lst = []\n",
        "val_loss_lst = []\n",
        "val_acc_lst = []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    train_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    count = 0\n",
        "    model.train()\n",
        "    for X_train, y_train in train_loader:\n",
        "        X_train, y_train = X_train.to(device), y_train.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        preds = model.forward(X_train)\n",
        "        loss = criterion(preds, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        train_loss += loss\n",
        "        train_acc += compute_accuracy(preds, y_train)\n",
        "        count += len(y_train)\n",
        "\n",
        "    train_loss /= len(train_loader)\n",
        "    train_loss_lst.append(train_loss.detach().cpu().numpy())\n",
        "    train_acc /= count\n",
        "    train_acc_lst.append(train_acc.detach().cpu().numpy())\n",
        "\n",
        "\n",
        "\n",
        "    val_loss = 0.0\n",
        "    val_acc = 0.0\n",
        "    count = 0\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for X_val, y_val in val_loader:\n",
        "            X_val, y_val = X_val.to(device), y_val.to(device)\n",
        "            preds = model.forward(X_val)\n",
        "            loss = criterion(preds, y_val)\n",
        "            val_loss += loss\n",
        "            val_acc += compute_accuracy(preds, y_val)\n",
        "            count += len(y_val)\n",
        "\n",
        "    val_loss /= len(test_loader)\n",
        "    val_loss_lst.append(val_loss.detach().cpu().numpy())\n",
        "    val_acc /= count\n",
        "    val_acc_lst.append(val_acc.detach().cpu().numpy())\n",
        "\n",
        "    print(f\"EPOCH {epoch + 1}/{epochs}, Train_Loss: {train_loss:.4f}, Train_Acc: {train_acc:.4f}, Validation Loss: {val_loss:.4f}, Val_Acc: {val_acc:.4f}\")"
      ],
      "metadata": {
        "trusted": true,
        "id": "hxeqEdxZvypU"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots(2, 2, figsize=(12, 10))\n",
        "ax[0, 0].plot(train_loss_lst, color='green')\n",
        "ax[0, 0].set(xlabel='Epoch', ylabel='Loss')\n",
        "ax[0, 0].set_title('Training Loss')\n",
        "\n",
        "ax[0, 1].plot(val_loss_lst, color='orange')\n",
        "ax[0, 1].set(xlabel='Epoch', ylabel='Loss')\n",
        "ax[0, 1].set_title('Validation Loss')\n",
        "\n",
        "ax[1, 0].plot(train_acc_lst, color='green')\n",
        "ax[1, 0].set(xlabel='Epoch', ylabel='Accuracy')\n",
        "ax[1, 0].set_title('Training Accuracy')\n",
        "\n",
        "ax[1, 1].plot(val_acc_lst, color='orange')\n",
        "ax[1, 1].set(xlabel='Epoch', ylabel='Accuracy')\n",
        "ax[1, 1].set_title('Validation Accuracy')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "trusted": true,
        "id": "Ll40rge5vypU"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "val_target = []\n",
        "val_predict = []\n",
        "\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "    for X_val, y_val in val_loader:\n",
        "        X_val = X_val.to(device)\n",
        "        y_val = y_val.to(device)\n",
        "        outputs = model(X_val)\n",
        "\n",
        "        val_predict.append(outputs.cpu())\n",
        "        val_target.append(y_val.cpu())\n",
        "\n",
        "    val_predict = torch.cat(val_predict)\n",
        "    val_target = torch.cat(val_target)\n",
        "    val_acc = (torch.argmax(val_predict, 1) == val_target).sum().item() / len(val_target)\n",
        "\n",
        "    print('Evaluation on val set:')\n",
        "    print(f'Accuracy: {val_acc}')"
      ],
      "metadata": {
        "trusted": true,
        "id": "ADYpVHFuvypV"
      },
      "outputs": [],
      "execution_count": null
    }
  ]
}