train-sfw-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3cbaf220",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from transformers import AutoTokenizer, AutoConfig, AutoModel\n",
    "from classifier import MistralForSequenceClassification\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "83fff176",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "texts, labels = [], []\n",
    "\n",
    "labels_sentiment = ['racist', 'religion insult','psychiatric or mental illness', 'sexist','harassment','hate','porn','safe for work']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1da77d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "\n",
    "with open('malaysian-sfw-dataset.jsonl') as f:\n",
    "    \n",
    "    for x in f:\n",
    "        data.append(json.loads(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f2a2d4d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4d1e1ca7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "209986"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for l in data:\n",
    "    if l['label'] not in labels_sentiment:\n",
    "        continue\n",
    "    if len(l['text']) > 5:\n",
    "        texts.append(l['text'])\n",
    "        labels.append(labels_sentiment.index(l['label']))\n",
    "        \n",
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cfe78473",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(\n",
    "    texts, labels, test_size = 0.1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5669ce11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Kaka simpati dgn Emilia..  Dia ni mangsa sexual abuse..  Kat thread kenamaan sana  ramai je yg mengata dia kak normah  kata dia kuat drama  berdendam lah  heran betul la  yg mengata tu perempuan kot',\n",
       "  3),\n",
       " ('Aku suka baca confession kat sini yg mostly cerita tentang betapa rendah dirinya mereka kerana gemuk la, hitam la, juling la, macam-macam. Ada yang cerita walaupun fizikal kurang menarik tapi masih ada yang berkenan lantas jatuh cinta hingga ke jinjang pelamin. Ada jugak yang tercari2 erti cinta sejati dek kerana rasa dirinya tidak sesempurna yang lain. Ermm penting sangat ke rupa dan fizikal nih?',\n",
       "  2),\n",
       " ('Bukan sekadar berak busuk, diorang geledah sampah, plastik sampah dekat dapur koyak-koyak, berlemoih woih dapur kitorang. Dengan tulang-tulang ayam diorang ratah sepah-sepah.',\n",
       "  4),\n",
       " ('Kene jackpot siaaa  dapat mamat badan athletic plus big ', 6),\n",
       " (\"Then he started giving me a b*****b. Takut, confused, semuanyalah. I didn't even c*m mungkin kerana terlalu terkejut. He on the other hand, did. Dia terus whatsapp aku selepas itu dan sesampainya di pintu hospital aku terus blocked number beliau.\",\n",
       "  4),\n",
       " ('Pada awal perkahwinan aku, aku selalu bermasam muka dengan suami. Buat salah sikit, aku punya marah macam beliau buat dosa besar. Suami balik dari kerja, mengagak2 aku mood baik ke tak baik. Psycho dibuatnya. Tapi suami aku punya kesabaran tahap tinggi. Kalau aku dapat bini cam diri aku sendiri, lama dah kena talak 3.',\n",
       "  2),\n",
       " ('OBE tu ape ye..  Out of body experience or astral projection. Boleh search thread berkaitan kat bod ni.',\n",
       "  7),\n",
       " ('Kat sini saya nak bagi beberapa pendapat peribadi saya mengenai pemikiran typical sesetengah orang melayu kita mengenai anak dara dan anak teruna. For sure pendapat saya ni tak akan sama dengan pendapat semua reader sebab setiap orang ada pemikiran yang berbeza. Jadi saya dah bersedia nak di \"bash\". Tapi takpelah, korang bashla sebab keyboard tu korang punya. Saya akan stay positive membacanya nanti sebab saya dah biasa handle orang typical. Hihihi',\n",
       "  3),\n",
       " ('komen korang nih, buat aku tersedak je.  udah2 lerr', 5),\n",
       " ('For the harmony of our marriage, my husband and I decided to live independently after getting married because I believe that as young people, we have our own plans for the future that sometimes our parents may not agree with, old school vs new school.',\n",
       "  7)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(texts[:10], labels[:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9f891192",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2, 3, 4, 5, 6, 7]),\n",
       " array([18673, 34037, 53324, 15647, 24716, 22924,   667, 39998]))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(labels, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ce2396bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "id2label = {i: label for i, label in enumerate(labels_sentiment)}\n",
    "label2id = {label: i for i, label in enumerate(labels_sentiment)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0fd65cb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "config = AutoConfig.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512')\n",
    "config.num_labels = len(set(labels))\n",
    "config.vocab = labels_sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2f0a2a9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of MistralForSequenceClassification were not initialized from the model checkpoint at mesolitica/malaysian-mistral-191M-MLM-512 and are newly initialized: ['score.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = MistralForSequenceClassification.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512', config = config)\n",
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c6604d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "41f5e2ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainable_parameters = [param for param in model.parameters() if param.requires_grad]\n",
    "trainer = torch.optim.AdamW(trainable_parameters, lr = 2e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "462dd7dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9450/9450 [36:28<00:00,  4.32it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, loss: 0.6004894968402134, dev_predicted: 0.8203784461152882\n",
      "[2024-05-31 12:55:25,516] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████▏   | 5798/9450 [22:21<13:00,  4.68it/s] "
     ]
    }
   ],
   "source": [
    "batch_size = 20\n",
    "epoch = 100\n",
    "\n",
    "\n",
    "best_dev_acc = -np.inf\n",
    "patient = 3\n",
    "current_patient = 0\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size))\n",
    "    losses = []\n",
    "    for i in pbar:\n",
    "        trainer.zero_grad()\n",
    "        x = train_X[i: i + batch_size]\n",
    "        y = np.array(train_Y[i: i + batch_size])\n",
    "        \n",
    "        padded = tokenizer(x, padding = 'longest', return_tensors = 'pt')\n",
    "\n",
    "        padded['labels'] = torch.from_numpy(y)\n",
    "        for k in padded.keys():\n",
    "            padded[k] = padded[k].cuda()\n",
    "        \n",
    "        padded.pop('token_type_ids', None)\n",
    "\n",
    "        loss, pred = model(**padded, return_dict = False)\n",
    "        loss.backward()\n",
    "        \n",
    "        grad_norm = torch.nn.utils.clip_grad_norm_(trainable_parameters, 5.0)\n",
    "        trainer.step()\n",
    "        losses.append(float(loss))\n",
    "        \n",
    "    dev_predicted = []\n",
    "    for i in range(0, len(test_X), batch_size):\n",
    "        x = test_X[i: i + batch_size]\n",
    "        y = np.array(test_Y[i: i + batch_size])\n",
    "        padded = tokenizer(x, padding = 'longest', return_tensors = 'pt')\n",
    "        padded['labels'] = torch.from_numpy(y)\n",
    "        for k in padded.keys():\n",
    "            padded[k] = padded[k].cuda()\n",
    "        \n",
    "        padded.pop('token_type_ids', None)\n",
    "\n",
    "        loss, pred = model(**padded, return_dict = False)\n",
    "        dev_predicted.append((pred.argmax(axis = 1).detach().cpu().numpy() == y).mean())\n",
    "        \n",
    "    dev_predicted = np.mean(dev_predicted)\n",
    "    \n",
    "    print(f'epoch: {e}, loss: {np.mean(losses)}, dev_predicted: {dev_predicted}')\n",
    "    \n",
    "    if dev_predicted >= best_dev_acc:\n",
    "        best_dev_acc = dev_predicted\n",
    "        current_patient = 0\n",
    "        model.save_pretrained('porn-mistral-mlm')\n",
    "    else:\n",
    "        current_patient += 1\n",
    "    \n",
    "    if current_patient >= patient:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d182b765",
   "metadata": {},
   "outputs": [],
   "source": [
    "padded = tokenizer(texts[:10], padding = True, return_tensors = 'pt')\n",
    "for k in padded.keys():\n",
    "    padded[k] = padded[k].cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "549e5416",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:00<00:00, 19.16it/s]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 20\n",
    "epoch = 100\n",
    "\n",
    "real_Y = []\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    x = test_X[i: i + batch_size]\n",
    "    y = np.array(test_Y[i: i + batch_size])\n",
    "    padded = tokenizer(x, padding = 'longest', return_tensors = 'pt')\n",
    "    padded['labels'] = torch.from_numpy(y)\n",
    "    for k in padded.keys():\n",
    "        padded[k] = padded[k].cuda()\n",
    "    padded.pop('token_type_ids', None)\n",
    "\n",
    "    loss, pred = model(**padded,return_dict=False)\n",
    "    real_Y.extend(pred.argmax(axis = 1).detach().cpu().numpy().tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "111e7288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    non porn    0.93939   0.93939   0.93939        33\n",
      "        porn    0.95556   0.95556   0.95556        45\n",
      "\n",
      "    accuracy                        0.94872        78\n",
      "   macro avg    0.94747   0.94747   0.94747        78\n",
      "weighted avg    0.94872   0.94872   0.94872        78\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, test_Y, target_names = config.vocab,\n",
    "        digits = 5\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ff24eb29",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-05-17 23:53:57.253255: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-05-17 23:54:01.407867: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "pipe = pipeline(\"text-classification\",\n",
    "                        tokenizer = tokenizer,\n",
    "                        model=model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "40351101",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['racist',\n",
       " 'religion insult',\n",
       " 'psychiatric or mental illness',\n",
       " 'sexist',\n",
       " 'harassment',\n",
       " 'informative',\n",
       " 'safe for work',\n",
       " 'hate']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9a368d51",
   "metadata": {},
   "outputs": [],
   "source": [
    "label2id = {'racist':0,'religion insult':1,'psychiatric or mental illness':2,\n",
    "            'sexist':3,'harassment':4,'informative':5,'safe for work':6,'hate':7}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9c5e4a39",
   "metadata": {},
   "outputs": [],
   "source": [
    "id2label = {0:'racist',1:'religion insult',2:'psychiatric or mental illness',\n",
    "            3:'sexist',4:'harassment',5:'informative',6:'safe for work',7:'hate'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "766e447c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.config.label2id = label2id\n",
    "model.config.id2label = id2label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "946c31d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'hate', 'score': 0.9171728491783142}]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe(['macam bodoh betul la engkau ni'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "65f08ac9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'racist', 'score': 0.9956732392311096}]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('melayu memang rasa tertindas padahal yahudi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b5bf9160",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'safe for work', 'score': 0.5028830170631409}]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2f235583",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'sexist', 'score': 0.9981914162635803}]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('perempuan memang tak pandai macam lelaki')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "addd5b73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'hate', 'score': 0.5682838559150696}]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('lelaki semua sampah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "38bbffb2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'hate', 'score': 0.49550363421440125}]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('hidup ni tak bermakna, takde sebab nak hidup lagi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "171d30f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'psychiatric or mental illness', 'score': 0.9957693815231323}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('aku nampak buaya nak makan aku, hantu takda kepala cakap dgn aku, aku tak boleh tido')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ce47f2b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'harassment', 'score': 0.5153989195823669}]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('perempuan gila tu nak suruh semua orang dgr cakap dia')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "0ede9452",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'psychiatric or mental illness', 'score': 0.9803110361099243}]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('saya nak bunuh diri, tak da makna hidup ni')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ffbb047d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'informative', 'score': 0.8735892176628113}]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('mak cik tu jual nasi lemak sedap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "d1bb8781",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'informative', 'score': 0.7604817748069763}]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe('amat penting untuk semua pelajar cakna akan isu semasa')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "0c2139ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RepoUrl('https://huggingface.co/malaysia-ai/malaysian-sfw-classifier', endpoint='https://huggingface.co', repo_type='model', repo_id='malaysia-ai/malaysian-sfw-classifier')"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from huggingface_hub import create_repo\n",
    "create_repo(\"malaysia-ai/malaysian-sfw-classifier\", repo_type=\"model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "094ff0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26450ac65ab3455a9bb743221a385857",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/1.30k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "056b0cf783a94583a3d814858ec3be3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/665M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/malaysia-ai/malaysian-sfw-classifier/commit/4196e7859db5d357a2fa57200c205b47fe939a69', commit_message='Upload MistralForSequenceClassification', commit_description='', oid='4196e7859db5d357a2fa57200c205b47fe939a69', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub('malaysia-ai/malaysian-sfw-classifier', safe_serialization = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "820f74dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c965875b6aa44c46b98bd731cef2a5c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/1.30k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/malaysia-ai/malaysian-sfw-classifier/commit/616f734b3ca3ea0e4f779acc961ad7349ccf08eb', commit_message='Upload tokenizer', commit_description='', oid='616f734b3ca3ea0e4f779acc961ad7349ccf08eb', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('malaysia-ai/malaysian-sfw-classifier', safe_serialization = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f90d87e7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}