LiTS_metric.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ed127b57",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "In the future `np.bool` will be defined as the corresponding NumPy scalar.\n",
      "In the future `np.bool` will be defined as the corresponding NumPy scalar.\n",
      "In the future `np.bool` will be defined as the corresponding NumPy scalar.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin C:\\Users\\user\\anaconda3\\envs\\ailab\\lib\\site-packages\\bitsandbytes\\libbitsandbytes_cuda116.dll\n",
      "CUDA SETUP: CUDA runtime path found: C:\\Users\\user\\anaconda3\\envs\\ailab\\bin\\cudart64_110.dll\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 116\n",
      "CUDA SETUP: Loading binary C:\\Users\\user\\anaconda3\\envs\\ailab\\lib\\site-packages\\bitsandbytes\\libbitsandbytes_cuda116.dll...\n"
     ]
    }
   ],
   "source": [
    "import torchio as tio\n",
    "import os\n",
    "import nibabel as nib\n",
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "\n",
    "from monai.metrics import compute_average_surface_distance, get_confusion_matrix\n",
    "from collections.abc import Sequence\n",
    "from monai.metrics.utils import get_mask_edges, get_surface_distance, ignore_background, prepare_spacing\n",
    "from monai.utils import convert_data_type, convert_to_tensor\n",
    "from tqdm import tqdm\n",
    "\n",
    "from scipy import ndimage\n",
    "#from scipy.ndimage import label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "372a6040",
   "metadata": {},
   "outputs": [],
   "source": [
    "# metric class\n",
    "class Metric:\n",
    "    def __init__(self, GT, PRED):\n",
    "        self.gt = GT\n",
    "        self.pred = PRED\n",
    "        \n",
    "        # metric list\n",
    "        self.DC_list = []\n",
    "        self.DG_numerator = []\n",
    "        self.DG_denominator = []\n",
    "        self.Jaccard_list = []\n",
    "        self.VOE_list = []\n",
    "        self.RVD_list = []\n",
    "        self.RAVD_list = []\n",
    "        self.ASSD_list = []\n",
    "        self.RMSD_list = []\n",
    "        self.MSSD_list = []\n",
    "        self.TP_list = []\n",
    "        self.TN_list = []\n",
    "        self.FP_list = []\n",
    "        self.FN_list = []\n",
    "        self.SEN_list = []\n",
    "        self.SPE_list = []\n",
    "        \n",
    "    def cal_DC_and_DG(self):\n",
    "        # inersection, union, dice 계산\n",
    "        intersection = (self.pred*self.gt).sum()\n",
    "        union = self.pred.sum() + self.gt.sum()\n",
    "        dice = 2*intersection/union\n",
    "        \n",
    "        self.DG_numerator.append(intersection)\n",
    "        self.DG_denominator.append(union)\n",
    "        self.DC_list.append(dice) # save\n",
    "    \n",
    "    def cal_Jaccard_and_VOE(self):\n",
    "        intersection = (self.pred*self.gt).sum()\n",
    "        union = self.pred.sum() + self.gt.sum() - (self.pred*self.gt).sum()\n",
    "        jaccard = intersection/union\n",
    "        \n",
    "        self.Jaccard_list.append(jaccard)\n",
    "        self.VOE_list.append(1-jaccard)\n",
    "    \n",
    "    # RAVD equation: by CotepRes-Net\n",
    "    def cal_RVD_and_RAVD(self):\n",
    "        numerator = self.pred.sum() - self.gt.sum()\n",
    "        denominator = self.pred.sum() # another paper mentioned, self.gt.sum()\n",
    "        RVD = numerator/denominator\n",
    "        \n",
    "        self.RVD_list.append(RVD)\n",
    "        self.RAVD_list.append(abs(RVD))\n",
    "    \n",
    "    # average symmetric surface distance (ASD or ASSD)\n",
    "    def cal_ASSD(self):\n",
    "        # monai module\n",
    "        # compute_average_surface_distance\n",
    "        assd = compute_average_surface_distance(self.pred, self.gt, symmetric=True)\n",
    "        \n",
    "        self.ASSD_list.append(assd)\n",
    "    \n",
    "    # root mean symmetric surface distance (RMSD)\n",
    "    def cal_RMSD(self):\n",
    "        rmsd = self.compute_root_mean_surface_distance(symmetric=True)\n",
    "        \n",
    "        self.RMSD_list.append(rmsd)\n",
    "        \n",
    "    # Maximum symmetric surface distance (MSD)\n",
    "    def cal_MSSD(self):\n",
    "        mssd = self.compute_maximum_surface_distance()\n",
    "        \n",
    "        self.MSSD_list.append(mssd)\n",
    "    \n",
    "    def cal_CONFUSION_MATRIX(self):\n",
    "        pred, gt = torch.Tensor(self.pred), torch.Tensor(self.gt)\n",
    "        confusion_matrix = get_confusion_matrix(pred, gt, include_background=False)\n",
    "        tp, fp, tn, fn = confusion_matrix[0,0,...] #  3D -> 1D\n",
    "        SEN = tp/(tp+fn)\n",
    "        SPE = tn/(tn+fp)\n",
    "        \n",
    "        self.TP_list.append(tp)\n",
    "        self.TN_list.append(tn)\n",
    "        self.FP_list.append(fp)\n",
    "        self.FN_list.append(fn)\n",
    "        \n",
    "        self.SEN_list.append(SEN)\n",
    "        self.SPE_list.append(SPE)\n",
    "    \n",
    "    def print_metric(self):\n",
    "        print(\"DC score(upper):\", self.DC_list[0])\n",
    "        #print(\"DG score(upper):\", self.DG_numerator[0]/self.DG_denominator[0])\n",
    "        print(\"Jaccard score(upper):\", self.Jaccard_list[0])\n",
    "        print(\"VOE score(lower):\", self.VOE_list[0])\n",
    "        #print(\"RVD score(lower):\", self.RVD_list[0])\n",
    "        print(\"RAVD score(lower):\", self.RAVD_list[0])\n",
    "        print(\"ASSD score(lower):\", self.ASSD_list[0].item())\n",
    "        print(\"RMSD score(lower):\", self.RMSD_list[0].item())\n",
    "        print(\"MSSD score(lower):\", self.MSSD_list[0].item())\n",
    "        print(\"SEN score(upper):\", self.SEN_list[0].item())\n",
    "        print(\"SPE score(upper):\", self.SPE_list[0].item())\n",
    "    \n",
    "    def save_metric(self):\n",
    "        return self.DC_list[0], self.Jaccard_list[0], self.VOE_list[0], self.RAVD_list[0], self.ASSD_list[0].item(), \\\n",
    "                    self.RMSD_list[0].item(), self.MSSD_list[0].item(), self.SEN_list[0].item(), self.SPE_list[0].item()\n",
    "    \n",
    "    def all_calculation(self):\n",
    "        print(\"######### Start! ######### \")\n",
    "        self.cal_DC_and_DG()\n",
    "        self.cal_Jaccard_and_VOE()\n",
    "        self.cal_RVD_and_RAVD()\n",
    "        self.cal_ASSD()\n",
    "        self.cal_RMSD()\n",
    "        self.cal_MSSD()\n",
    "        self.cal_CONFUSION_MATRIX()\n",
    "        print(\"######### Done! ######### \")\n",
    "    \n",
    "    ############### Refereces by Monai\n",
    "    def compute_root_mean_surface_distance(\n",
    "        self,\n",
    "        include_background = False,\n",
    "        symmetric = True,\n",
    "        distance_metric = \"euclidean\",\n",
    "        spacing = None):\n",
    "\n",
    "        if not include_background:\n",
    "            y_pred, y = ignore_background(y_pred=self.pred, y=self.gt)\n",
    "        else:\n",
    "            y_pred, y = self.pred.copy(), self.gt.copy()\n",
    "\n",
    "        y_pred = convert_data_type(y_pred, output_type=torch.Tensor, dtype=torch.float)[0]\n",
    "        y = convert_data_type(y, output_type=torch.Tensor, dtype=torch.float)[0]\n",
    "\n",
    "        if y.shape != y_pred.shape:\n",
    "            raise ValueError(f\"y_pred and y should have same shapes, got {y_pred.shape} and {y.shape}.\")\n",
    "\n",
    "        batch_size, n_class = y_pred.shape[:2]\n",
    "        asd = torch.empty((batch_size, n_class), dtype=torch.float32, device=y_pred.device)\n",
    "\n",
    "        img_dim = y_pred.ndim - 2\n",
    "        spacing_list = prepare_spacing(spacing=spacing, batch_size=batch_size, img_dim=img_dim)\n",
    "\n",
    "        for b, c in np.ndindex(batch_size, n_class):\n",
    "            _, distances, _ = self.get_edge_surface_distance(\n",
    "                y_pred[b, c],\n",
    "                y[b, c],\n",
    "                distance_metric=distance_metric,\n",
    "                spacing=spacing_list[b],\n",
    "                symmetric=symmetric,\n",
    "                class_index=c,\n",
    "            )\n",
    "            surface_distance = torch.cat(distances)\n",
    "            asd[b, c] = torch.tensor(np.nan) if surface_distance.shape == (0,) else torch.sqrt(torch.mean((surface_distance)**2))\n",
    "            \n",
    "        return convert_data_type(asd, output_type=torch.Tensor, device=y_pred.device, dtype=torch.float)[0]\n",
    "    \n",
    "    # Make MSSD\n",
    "    def compute_maximum_surface_distance(\n",
    "        self,\n",
    "        include_background = False,\n",
    "        symmetric = True,\n",
    "        distance_metric = \"euclidean\",\n",
    "        spacing = None):\n",
    "\n",
    "        if not include_background:\n",
    "            y_pred, y = ignore_background(y_pred=self.pred, y=self.gt)\n",
    "        else:\n",
    "            y_pred, y = self.pred.copy(), self.gt.copy()\n",
    "\n",
    "        y_pred = convert_data_type(y_pred, output_type=torch.Tensor, dtype=torch.float)[0]\n",
    "        y = convert_data_type(y, output_type=torch.Tensor, dtype=torch.float)[0]\n",
    "\n",
    "        if y.shape != y_pred.shape:\n",
    "            raise ValueError(f\"y_pred and y should have same shapes, got {y_pred.shape} and {y.shape}.\")\n",
    "\n",
    "        batch_size, n_class = y_pred.shape[:2]\n",
    "        asd = torch.empty((batch_size, n_class), dtype=torch.float32, device=y_pred.device)\n",
    "\n",
    "        img_dim = y_pred.ndim - 2\n",
    "        spacing_list = prepare_spacing(spacing=spacing, batch_size=batch_size, img_dim=img_dim)\n",
    "\n",
    "        for b, c in np.ndindex(batch_size, n_class):\n",
    "            _, distances, _ = self.get_edge_surface_distance(\n",
    "                y_pred[b, c],\n",
    "                y[b, c],\n",
    "                distance_metric=distance_metric,\n",
    "                spacing=spacing_list[b],\n",
    "                symmetric=symmetric,\n",
    "                class_index=c,\n",
    "            )\n",
    "            surface_distance = torch.cat(distances)\n",
    "            asd[b, c] = torch.tensor(np.nan) if surface_distance.shape == (0,) else max(surface_distance)\n",
    "            \n",
    "        return convert_data_type(asd, output_type=torch.Tensor, device=y_pred.device, dtype=torch.float)[0]\n",
    "    \n",
    "    # get_edge_surface_distance\n",
    "    def get_edge_surface_distance(\n",
    "        self,\n",
    "        y_pred,\n",
    "        y,\n",
    "        distance_metric = \"euclidean\",\n",
    "        spacing: \"int | float | np.ndarray | Sequence[int | float] | None\" = None,\n",
    "        use_subvoxels: bool = False,\n",
    "        symmetric: bool = False,\n",
    "        class_index: int = -1):\n",
    "        \n",
    "        edges_spacing = None\n",
    "        if use_subvoxels:\n",
    "            edges_spacing = spacing if spacing is not None else ([1] * len(y_pred.shape))\n",
    "        \n",
    "        (edges_pred, edges_gt, *areas) = get_mask_edges(y_pred, y, crop=True, spacing=edges_spacing, always_return_as_numpy=False)\n",
    "        \n",
    "        if symmetric:\n",
    "            distances = (\n",
    "                get_surface_distance(edges_pred, edges_gt, distance_metric, spacing),\n",
    "                get_surface_distance(edges_gt, edges_pred, distance_metric, spacing),\n",
    "            )  # type: ignore\n",
    "        else:\n",
    "            distances = (get_surface_distance(edges_pred, edges_gt, distance_metric, spacing),)  # type: ignore\n",
    "            \n",
    "        return convert_to_tensor(((edges_pred, edges_gt), distances, tuple(areas)), device=y_pred.device)  # type: ignore[no-any-return]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f171ac6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculation function\n",
    "def calculation_all_metric(GT, PRED):\n",
    "    metric = Metric(GT, PRED)\n",
    "    metric.all_calculation()\n",
    "    metric.print_metric()\n",
    "    \n",
    "    return metric.save_metric()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d4ab89a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_all_but_the_largest_connected_component(image: np.ndarray, for_which_classes: list, volume_per_voxel: float,\n",
    "                                                   minimum_valid_object_size: dict = None):\n",
    "    \"\"\"\n",
    "    removes all but the largest connected component, individually for each class\n",
    "    :param image:\n",
    "    :param for_which_classes: can be None. Should be list of int. Can also be something like [(1, 2), 2, 4].\n",
    "    Here (1, 2) will be treated as a joint region, not individual classes (example LiTS here we can use (1, 2)\n",
    "    to use all foreground classes together)\n",
    "    :param minimum_valid_object_size: Only objects larger than minimum_valid_object_size will be removed. Keys in\n",
    "    minimum_valid_object_size must match entries in for_which_classes\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    if for_which_classes is None:\n",
    "        for_which_classes = np.unique(image)\n",
    "        for_which_classes = for_which_classes[for_which_classes > 0]\n",
    "\n",
    "    assert 0 not in for_which_classes, \"cannot remove background\"\n",
    "    largest_removed = {}\n",
    "    kept_size = {}\n",
    "    for c in for_which_classes:\n",
    "        if isinstance(c, (list, tuple)):\n",
    "            c = tuple(c)  # otherwise it cant be used as key in the dict\n",
    "            mask = np.zeros_like(image, dtype=bool)\n",
    "            for cl in c:\n",
    "                mask[image == cl] = True\n",
    "        else:\n",
    "            mask = image == c\n",
    "        # get labelmap and number of objects\n",
    "        lmap, num_objects = ndimage.label(mask.astype(int))\n",
    "\n",
    "        # collect object sizes\n",
    "        object_sizes = {}\n",
    "        for object_id in range(1, num_objects + 1):\n",
    "            object_sizes[object_id] = (lmap == object_id).sum() * volume_per_voxel\n",
    "\n",
    "        largest_removed[c] = None\n",
    "        kept_size[c] = None\n",
    "\n",
    "        if num_objects > 0:\n",
    "            # we always keep the largest object. We could also consider removing the largest object if it is smaller\n",
    "            # than minimum_valid_object_size in the future but we don't do that now.\n",
    "            maximum_size = max(object_sizes.values())\n",
    "            kept_size[c] = maximum_size\n",
    "\n",
    "            for object_id in range(1, num_objects + 1):\n",
    "                # we only remove objects that are not the largest\n",
    "                if object_sizes[object_id] != maximum_size:\n",
    "                    # we only remove objects that are smaller than minimum_valid_object_size\n",
    "                    remove = True\n",
    "                    if minimum_valid_object_size is not None:\n",
    "                        remove = object_sizes[object_id] < minimum_valid_object_size[c]\n",
    "                    if remove:\n",
    "                        image[(lmap == object_id) & mask] = 0\n",
    "                        if largest_removed[c] is None:\n",
    "                            largest_removed[c] = object_sizes[object_id]\n",
    "                        else:\n",
    "                            largest_removed[c] = max(largest_removed[c], object_sizes[object_id])\n",
    "    return image, largest_removed, kept_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "31f1e50f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def post_processing(data, whole=False):\n",
    "    \n",
    "    \n",
    "    processed, largest_removed, kept_size = remove_all_but_the_largest_connected_component(\n",
    "        image=data,\n",
    "        for_which_classes=[1],\n",
    "        volume_per_voxel=3,\n",
    "        minimum_valid_object_size=None  # 이 인자는 생략하거나 None으로 설정할 수 있dma\n",
    "        )\n",
    "\n",
    "    processed = ndimage.median_filter(processed, size=3)\n",
    "\n",
    "    #klcc_transform=KeepLargestConnectedComponent(applied_labels=[1], is_onehot=None, independent=False, connectivity=True, num_components=1)\n",
    "    \n",
    "    #processed=klcc_transform(data)\n",
    "\n",
    "    #erosioned = ndimage.binary_erosion(processed, iterations=3)\n",
    "    \n",
    "    #dilationed = ndimage.binary_dilation(erosioned, iterations=3)\n",
    "    \n",
    "    #processed = np.where(dilationed == 0,0,1)\n",
    "    \n",
    "    return processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0f9abead",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset names(3Dircadb, LiTS, Sliver):LiTS\n",
      "Extension: .nii\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# Setting\n",
    "data = input(\"Dataset names(3Dircadb, LiTS, Sliver):\")\n",
    "if data in ['3Dircadb', 'LiTS', 'Sliver']:\n",
    "    if data == 'LiTS':\n",
    "        extension = '.nii'\n",
    "    elif data == '3Dircadb':\n",
    "        extension = '.nii.gz'\n",
    "    else:\n",
    "        extension = '.mhd'\n",
    "    print(\"Extension:\", extension)\n",
    "    \n",
    "    # define path (설정)\n",
    "    gt_folder = './DATASET_Synapse/unetr_pp_raw/unetr_pp_raw_data/Task02_Synapse/Task002_Synapse/seg_gt/' + data + '/'\n",
    "    pred_folder = './output_synapse/3d_fullres/Task002_Synapse/unetr_pp_trainer_synapse__unetr_pp_Plansv2.1/fold_4/validation_raw/'\n",
    "    # 256x256\n",
    "\n",
    "    # make nii files\n",
    "    flag_post = True\n",
    "    pred_name = os.listdir(pred_folder)\n",
    "    pred_name = [nii for nii in pred_name if extension in nii]\n",
    "    print(len(pred_name)) # LiTS: 20\n",
    "\n",
    "    # 기타 변수 (rotation 안 되어 있다면 활용)\n",
    "    tmp_lst=['volume-15.nii', 'volume-18.nii', 'volume-28.nii', 'volume-3.nii', 'volume-33.nii', 'volume-37.nii', 'volume-42.nii', 'volume-47.nii', 'volume-5.nii', 'volume-54.nii', 'volume-70.nii', 'volume-73.nii', 'volume-80.nii']\n",
    "\n",
    "else:\n",
    "    print(\"No dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d4795074",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]\n"
     ]
    }
   ],
   "source": [
    "metric_df = pd.DataFrame(columns=['LiTS_id', 'DC', 'Jaccard', 'VOE', 'RAVD', 'ASSD', 'RMSD', 'MSSD', 'SEN', 'SPE'])\n",
    "\n",
    "for i in tqdm(range(len(pred_name))):\n",
    "    resize = tio.Resize((256, 256, -1))\n",
    "    pred = tio.ScalarImage(os.path.join(pred_folder, pred_name[i]))\n",
    "    if data == 'LiTS':\n",
    "        gt = tio.ScalarImage(os.path.join(gt_folder, pred_name[i].replace('volume', 'segmentation')))   \n",
    "    elif data == 'Sliver':\n",
    "        gt = tio.ScalarImage(os.path.join(gt_folder, pred_name[i].replace('orig', 'seg')[:-4]+'.mhd'))\n",
    "    else:\n",
    "        gt = tio.ScalarImage(os.path.join(gt_folder, pred_name[i].replace('volume', 'segmentation')+'.gz')) # 3Dircadb\n",
    "\n",
    "    gt.data = np.where(gt.data >= 1, 1, 0)\n",
    "    if len(np.unique(gt)) != 2: # error check\n",
    "        raise Exception(\"No label\")\n",
    "\n",
    "    pred = np.array(pred)\n",
    "    pred = pred[0]\n",
    "    if flag_post: # post-processing\n",
    "        pred = post_processing(pred)\n",
    "\n",
    "    gt = np.array(gt)\n",
    "    gt = resize(gt)\n",
    "    gt = np.transpose(gt[0], (2,1,0))\n",
    "    if pred_name[i] in tmp_lst:\n",
    "    #if True: # 3Dircard\n",
    "        gt = torch.flip(torch.Tensor(gt),(2,))\n",
    "        gt = np.array(gt)\n",
    "    pred = pred[np.newaxis, np.newaxis, ]\n",
    "    gt = gt[np.newaxis, np.newaxis, ]\n",
    "\n",
    "    #print(pred.shape) # 5D (batch, class, axial, X, Y)\n",
    "    #print(gt.shape) # 5D\n",
    "\n",
    "    # calculation metric\n",
    "    dc, jac, voe, ravd, assd, rmsd, mssd, sen, spe = calculation_all_metric(post_pred, gt)\n",
    "\n",
    "    # save\n",
    "    metric_df.loc[i] = [pred_name[i], dc, jac, voe, ravd, assd, rmsd, mssd, sen, spe]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "48f41bd3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LiTS_id    NaN\n",
       "DC         NaN\n",
       "Jaccard    NaN\n",
       "VOE        NaN\n",
       "RAVD       NaN\n",
       "ASSD       NaN\n",
       "RMSD       NaN\n",
       "MSSD       NaN\n",
       "SEN        NaN\n",
       "SPE        NaN\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show result\n",
    "metric_df.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "474854b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('./excel_result', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "49cfa2d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df save\n",
    "metric_df.loc[-1] = metric_df.mean(axis=0)\n",
    "metric_df.to_csv('./excel_result/'+data+'_whole_result.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "016d13fb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}