04b_tfrecord.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf \n",
    "import numpy as np\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "import PIL\n",
    "import gc\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numpy version : 1.18.1\n",
      "Pandas version : 1.0.3\n",
      "Tensorflow version : 2.2.0\n",
      "Pillow version : 5.4.1\n"
     ]
    }
   ],
   "source": [
    "print(f'Numpy version : {np.__version__}')\n",
    "print(f'Pandas version : {pd.__version__}')\n",
    "print(f'Tensorflow version : {tf.__version__}')\n",
    "print(f'Pillow version : {PIL.__version__}')"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "csv-with-cleaned-ocr-text  shopee-product-detection-student\r\n"
     ]
    }
   ],
   "source": [
    "!ls /kaggle/input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_parquet('/kaggle/input/csv-with-cleaned-ocr-text/train.parquet', engine='pyarrow').sort_values(\"filename\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "      <th>words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fd663cf2b6e1d7b02938c6aaae0a32d2.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c7fd77508a8c355eaab0d4e10efd6b15.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>127f3e6d6e3491b2459812353f33a913.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5ca4f2da11eda083064e6c36f37eeb81.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>46d681a542f2c71be017eef6aae23313.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12181</th>\n",
       "      <td>5ba958eacb23cd7d1673bad4dae55784.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12182</th>\n",
       "      <td>efbe41a1c2b666b70e337e438559808b.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12183</th>\n",
       "      <td>79fdaa5ac5ba10dbe8004cabd8c35eb3.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12184</th>\n",
       "      <td>ac3d136124617637a05ba66694e381ef.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12185</th>\n",
       "      <td>7ef61d7cfbad9cfe2db4f64560e3dddd.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12186 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   filename  category  \\\n",
       "0      fd663cf2b6e1d7b02938c6aaae0a32d2.jpg        43   \n",
       "1      c7fd77508a8c355eaab0d4e10efd6b15.jpg        43   \n",
       "2      127f3e6d6e3491b2459812353f33a913.jpg        43   \n",
       "3      5ca4f2da11eda083064e6c36f37eeb81.jpg        43   \n",
       "4      46d681a542f2c71be017eef6aae23313.jpg        43   \n",
       "...                                     ...       ...   \n",
       "12181  5ba958eacb23cd7d1673bad4dae55784.jpg        43   \n",
       "12182  efbe41a1c2b666b70e337e438559808b.jpg        43   \n",
       "12183  79fdaa5ac5ba10dbe8004cabd8c35eb3.jpg        43   \n",
       "12184  ac3d136124617637a05ba66694e381ef.jpg        43   \n",
       "12185  7ef61d7cfbad9cfe2db4f64560e3dddd.jpg        43   \n",
       "\n",
       "                                                   words  \n",
       "0      [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "1      [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "2      [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "3      [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "4      [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "...                                                  ...  \n",
       "12181  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "12182  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "12183  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "12184  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "12185  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "\n",
       "[12186 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_parquet('/kaggle/input/csv-with-cleaned-ocr-text/test.parquet', engine='pyarrow')\n",
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Create TFRecord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _bytes_feature(value):\n",
    "  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n",
    "  if isinstance(value, type(tf.constant(0))):\n",
    "    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n",
    "  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n",
    "\n",
    "def _float_feature(value):\n",
    "  \"\"\"Returns a float_list from a float / double.\"\"\"\n",
    "  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n",
    "\n",
    "def _list_float_feature(value):\n",
    "  \"\"\"Returns a float_list from a float / double.\"\"\"\n",
    "  return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n",
    "\n",
    "def _int64_feature(value):\n",
    "  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n",
    "  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n",
    "\n",
    "def _list_int64_feature(value):\n",
    "  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n",
    "  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESIZE_WIDTH = 512\n",
    "RESIZE_HEIGHT = 512\n",
    "\n",
    "TFRECORD_MAX_SIZE = 80 * 1024 * 1024 # 80 MB\n",
    "\n",
    "TOTAL_IMAGES = len(df_train.index)\n",
    "# TOTAL_IMAGES = len(df_test.index)\n",
    "\n",
    "# part 1 : 0:TOTAL_IMAGES // 2 (train)\n",
    "# part 2 : TOTAL_IMAGES // 2:TOTAL_IMAGES (train) [CURRENT]\n",
    "# part 3 : 0:TOTAL_IMAGES (test)\n",
    "START_INDEX = TOTAL_IMAGES // 2\n",
    "END_INDEX = TOTAL_IMAGES\n",
    "\n",
    "BATCH_IMAGE = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_tfrecord(index, df):\n",
    "    index = str(index).zfill(3)\n",
    "    curr_file = f\"train-{index}.tfrecords\"\n",
    "    writer = tf.io.TFRecordWriter(curr_file)\n",
    "    for index, row in df.iterrows():\n",
    "        category_str = str(row['category']).zfill(2)\n",
    "\n",
    "        image = f'/kaggle/input/shopee-product-detection-student/train/train/train/{category_str}/{row[\"filename\"]}'\n",
    "        img = open(image, 'rb')\n",
    "        img_read = img.read()\n",
    "        image_decoded = tf.image.decode_jpeg(img_read, channels=3)\n",
    "        resized_img = tf.image.resize_with_pad(image_decoded,target_width=RESIZE_WIDTH,target_height=RESIZE_HEIGHT,method=tf.image.ResizeMethod.BILINEAR)\n",
    "        resized_img = tf.cast(resized_img,tf.uint8)\n",
    "        resized_img = tf.io.encode_jpeg(resized_img)\n",
    "\n",
    "        feature = {\n",
    "            'filename': _bytes_feature(tf.compat.as_bytes(row['filename'])),\n",
    "            'label': _int64_feature(row['category']),\n",
    "            'words': _list_float_feature(row['words']),\n",
    "            'image': _bytes_feature(resized_img),\n",
    "            'height' : _int64_feature(RESIZE_HEIGHT),\n",
    "            'width' : _int64_feature(RESIZE_WIDTH)\n",
    "        }\n",
    "        example = tf.train.Example(features=tf.train.Features(feature=feature))\n",
    "        writer.write(example.SerializeToString())\n",
    "    writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create TFRecords #52\n",
      "Create TFRecords #53\n",
      "Create TFRecords #54\n",
      "Create TFRecords #55\n",
      "Create TFRecords #56\n",
      "Create TFRecords #57\n",
      "Create TFRecords #58\n",
      "Create TFRecords #59\n",
      "Create TFRecords #60\n",
      "Create TFRecords #61\n",
      "Create TFRecords #62\n",
      "Create TFRecords #63\n",
      "Create TFRecords #64\n",
      "Create TFRecords #65\n",
      "Create TFRecords #66\n",
      "Create TFRecords #67\n",
      "Create TFRecords #68\n",
      "Create TFRecords #69\n",
      "Create TFRecords #70\n",
      "Create TFRecords #71\n",
      "Create TFRecords #72\n",
      "Create TFRecords #73\n",
      "Create TFRecords #74\n",
      "Create TFRecords #75\n",
      "Create TFRecords #76\n",
      "Create TFRecords #77\n",
      "Create TFRecords #78\n",
      "Create TFRecords #79\n",
      "Create TFRecords #80\n",
      "Create TFRecords #81\n",
      "Create TFRecords #82\n",
      "Create TFRecords #83\n",
      "Create TFRecords #84\n",
      "Create TFRecords #85\n",
      "Create TFRecords #86\n",
      "Create TFRecords #87\n",
      "Create TFRecords #88\n",
      "Create TFRecords #89\n",
      "Create TFRecords #90\n",
      "Create TFRecords #91\n",
      "Create TFRecords #92\n",
      "Create TFRecords #93\n",
      "Create TFRecords #94\n",
      "Create TFRecords #95\n",
      "Create TFRecords #96\n",
      "Create TFRecords #97\n",
      "Create TFRecords #98\n",
      "Create TFRecords #99\n",
      "Create TFRecords #100\n",
      "Create TFRecords #101\n",
      "Create TFRecords #102\n",
      "Create TFRecords #103\n"
     ]
    }
   ],
   "source": [
    "for i in range(START_INDEX, END_INDEX, BATCH_IMAGE):\n",
    "    print(f'Create TFRecords #{i // BATCH_IMAGE + 1}')\n",
    "    if i + BATCH_IMAGE < END_INDEX:\n",
    "        create_tfrecord(i // BATCH_IMAGE + 1, df_train.loc[i:i+BATCH_IMAGE])\n",
    "    else:\n",
    "        create_tfrecord(i // BATCH_IMAGE + 1, df_train.loc[i:END_INDEX])\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 4.8G\r\n",
      "drwxr-xr-x 2 root root 4.0K Jul  1 09:25 .\r\n",
      "drwxr-xr-x 6 root root 4.0K Jul  1 09:10 ..\r\n",
      "---------- 1 root root 5.3K Jul  1 09:10 __notebook__.ipynb\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:11 train-052.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:11 train-053.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:12 train-054.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:12 train-055.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:12 train-056.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:12 train-057.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:13 train-058.tfrecords\r\n",
      "-rw-r--r-- 1 root root  92M Jul  1 09:13 train-059.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:13 train-060.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:14 train-061.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:14 train-062.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:14 train-063.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:14 train-064.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:15 train-065.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:15 train-066.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:15 train-067.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:15 train-068.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:16 train-069.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:16 train-070.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:16 train-071.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:17 train-072.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:17 train-073.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:17 train-074.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:17 train-075.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:18 train-076.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:18 train-077.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:18 train-078.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:19 train-079.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:19 train-080.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:19 train-081.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:19 train-082.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:20 train-083.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:20 train-084.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:20 train-085.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:20 train-086.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:21 train-087.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:21 train-088.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:21 train-089.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:22 train-090.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:22 train-091.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:22 train-092.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:22 train-093.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:23 train-094.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:23 train-095.tfrecords\r\n",
      "-rw-r--r-- 1 root root  95M Jul  1 09:23 train-096.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:23 train-097.tfrecords\r\n",
      "-rw-r--r-- 1 root root  96M Jul  1 09:24 train-098.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:24 train-099.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:24 train-100.tfrecords\r\n",
      "-rw-r--r-- 1 root root  94M Jul  1 09:25 train-101.tfrecords\r\n",
      "-rw-r--r-- 1 root root  93M Jul  1 09:25 train-102.tfrecords\r\n",
      "-rw-r--r-- 1 root root  44M Jul  1 09:25 train-103.tfrecords\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lah"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}