1_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "260d7ebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import os\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from tqdm.auto import tqdm\n",
    "from glob import glob\n",
    "import matplotlib.pyplot as plt\n",
    "from types import SimpleNamespace\n",
    "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
    "pd.options.display.max_colwidth = 1000\n",
    "pd.options.display.max_rows = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "id": "c15e3ffa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8720, 3) (17028, 2) (17028, 2) (69, 4)\n"
     ]
    }
   ],
   "source": [
    "BASE_DIR = \"./data\"\n",
    "df_submission = pd.read_csv(f\"{BASE_DIR}/submission.csv\")\n",
    "df_train = pd.read_csv(f\"{BASE_DIR}/train.csv\")\n",
    "df_test = pd.read_csv(f\"{BASE_DIR}/test.csv\")\n",
    "df_synset_meaning = pd.read_csv(f\"{BASE_DIR}/synset_meaning.csv\")\n",
    "df_synset_meaning[\"word_len\"] = df_synset_meaning.word.apply(len)\n",
    "print(df_train.shape, df_test.shape, df_submission.shape, df_synset_meaning.shape,)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "045bffd5",
   "metadata": {},
   "source": [
    "Fixing wrong labels:\n",
    "\n",
    "`шоронгоос гармагц#0000005434 эрүүлr агаар өөдөөс нь угтан`\n",
    "\n",
    "`эрүүлr => эрүүл`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "cdad0906",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.loc[:, \"text\"] = df_train.loc[:, \"text\"].str.replace(\"r\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "id": "b3eeee26",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_consecutive_item_error(df: pd.DataFrame, intersection_threshold=0.24):\n",
    "    new_texts = []\n",
    "    latest_similar = df.loc[0, \"text\"]\n",
    "    potential_bad_dict = {}\n",
    "    for i, row in tqdm(df.iterrows(), total=df.shape[0]):\n",
    "        text = row.text\n",
    "        tokens_prev = latest_similar.split()\n",
    "        tokens_cur  = text.split()\n",
    "\n",
    "        intersection = set(tokens_cur).intersection(tokens_prev)\n",
    "        union = set(tokens_cur).union(tokens_prev)\n",
    "\n",
    "        if len(intersection) / len(union) > intersection_threshold:\n",
    "            keys = list(potential_bad_dict.keys())\n",
    "            keys.reverse()\n",
    "\n",
    "            new_text = text\n",
    "            for key in keys:\n",
    "                new_text =new_text.replace(key, \" \".join(potential_bad_dict[key]))\n",
    "            new_texts.append((i, row.text_id, new_text))\n",
    "            \n",
    "            df.loc[i, \"text\"] = new_text #     <------- Replacing\n",
    "        else:\n",
    "            potential_bad_dict = {}\n",
    "            latest_similar = text\n",
    "\n",
    "        query_word = None\n",
    "        for tok in tokens_cur:\n",
    "            if \"#\" in tok:\n",
    "                query_word = tok.split(\"#\")[0]\n",
    "                continue\n",
    "            if query_word:\n",
    "                potential_bad_dict[query_word + tok] = [query_word, tok]\n",
    "                break\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "id": "d4d0203d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d589e0330a0e413fa7df07018b8f7a52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8720.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c0185a6536f4e21b6165506df064d26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17028.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((8720, 3), (17028, 2))"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = preprocess_consecutive_item_error(df_train)\n",
    "df_test = preprocess_consecutive_item_error(df_test)\n",
    "df_train.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "82f7e203",
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_labeling_error(df, index=101, old=\"улаан,мөнгө\", new=\"улаан, мөнгө\"):\n",
    "    df.loc[index, \"text\"] = df.loc[index, \"text\"].replace(old, new)\n",
    "    \n",
    "for a in [\n",
    "    [df_train, 101, \"улаан,мөнгө\", \"улаан, мөнгө\"],\n",
    "    [df_train, 666, \"хугацааны шийдвэрлэх үе#0000000667 шатанд\", \"хугацааны шийдвэрлэх үе шатанд#0000000667\"],\n",
    "    [df_train, 2878, \"үүднээссум\", \"үүднээс сум\"],\n",
    "    [df_train, 8547, \"гэрийнбараа\", \"гэрийн бараа\"],\n",
    "    [df_train, 8714, \"Cp\", \"сар\"],\n",
    "    [df_train, 8715, \"Cp\", \"сар\"],\n",
    "]:\n",
    "    replace_labeling_error(*a)\n",
    "    \n",
    "for a in [\n",
    "    [df_test, 485, \"хийх үе#0000009206 шатанд\", \"хийх үе шатанд#0000009206\"],\n",
    "    [df_test, 1764, \"тавирч,нүд#0000010485\", \"тавирч, нүд#0000010485\"],\n",
    "    [df_test, 2353, \"төвбайр#0000011074\", \"төв байр#0000011074\"],\n",
    "    [df_test, 2961, \"улсcтөрийн#0000011682\", \"улс төрийн#0000011682\"],\n",
    "    [df_test, 13964, \"гэрбараагаа#0000022685\", \"гэр бараагаа#0000022685\"],\n",
    "    [df_test, 10690, \"төвтэйтөрийн#0000019411\", \"төвтэй төрийн#0000019411\"],\n",
    "    [df_test, 6635, \"нүүр царайг#0000015356\", \"нүүр#0000015356 царайг\"],\n",
    "    \n",
    "]:\n",
    "    replace_labeling_error(*a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "id": "3e117060",
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "# Dictionary prepration\n",
    "dict_synset_meaning = collections.defaultdict(dict)\n",
    "for row in df_synset_meaning.itertuples():\n",
    "    dict_synset_meaning[row.word][row.synset_id] = row.meaning.lower()\n",
    "synset_id2word = {row.synset_id:row.word for row in df_synset_meaning.itertuples()}\n",
    "df_train[\"synset_word\"] = df_train.synset_id.map(synset_id2word)\n",
    "unique_synset = set(df_synset_meaning.word.unique().tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "a836bd0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Өнөөдөр ажлаасаа авахуулчихлаа#0000009274'"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# , ',', '.', '!', '?'\n",
    "\n",
    "replace_by_empty = [\n",
    "    \"...\", \"..\", \"\\\",\", '\"', '%', '&', \"'\", '(', ')',\n",
    "    '*', '+', '-', '/', ':', ';', '<', '>',\n",
    "    '[', '\\\\', ']', '{', '}', '¬', '‘', '’', '№', \"r\"\n",
    "]\n",
    "\n",
    "replace_by_space = [\n",
    "    '\\n', \n",
    "]\n",
    "\n",
    "def preprocess(text:str):\n",
    "    # text = text.lower()\n",
    "    for c in replace_by_empty:\n",
    "        text = text.replace(c, \"\")\n",
    "    for c in replace_by_space:\n",
    "        text = text.replace(c, \" \")\n",
    "    return text\n",
    "\n",
    "text = \"Ой, модны аж үйлдвэрийн салбарт ойн баялгийг арчлан хамгаалах, зөв зохистой ашиглах тухай хууль тогтоомжийн заалтыг цөөнгүй зөрчиж байгаагаа төлөвлөгөө биелүүлэх явцад зайлшгүй тийм хохирол, зөрчил гарч#0000002121 байх юм шиг өрөөсгөлөөр ойлгож, тууштай тэмцэл хийхгүй байна\"\n",
    "preproces(text)\n",
    "\n",
    "def preprocess_test(text:str):\n",
    "    for c in [\"...\", \"..\", \" \", \"...\\\"\", \"!\\\"\"]:\n",
    "        text = text.replace(c+\"#\", \"#\")\n",
    "    \n",
    "    return preprocess(text)\n",
    "preproces_test('Өнөөдөр ажлаасаа авахуулчихлаа...#0000009274')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "id": "3f774b04",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_similar_by_start(token1:str, token2:str) -> int:\n",
    "    count = 0\n",
    "    for i in range(min(len(token1), len(token2))):\n",
    "        if token1[i] == token2[i]:\n",
    "            count += 1\n",
    "        else:\n",
    "            break\n",
    "    return count / min(len(token1), len(token2))\n",
    "\n",
    "def find_closest(query_token):\n",
    "    query_token = query_token.lower()\n",
    "    if query_token in unique_synset:\n",
    "        return query_token, 1.\n",
    "    \n",
    "    best = float(\"-inf\")\n",
    "    best_syn = None\n",
    "    for syn in unique_synset:\n",
    "        d = calculate_similar_by_start(query_token, syn)\n",
    "        if d > best:\n",
    "            best_syn = syn\n",
    "            best = d\n",
    "            \n",
    "    return best_syn, best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "id": "cca1d9b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_closest_meaning(text, best_syn):\n",
    "    text = handle_special_char(text) # handling \"#id\"\n",
    "    \n",
    "    best = float(\"-inf\")\n",
    "    best_idx = None\n",
    "    best_meaning = None\n",
    "    for idx, meaning in dict_synset_meaning[best_syn].items():\n",
    "        d = count_intersections(preprocess(text), preprocess(meaning))\n",
    "        if d > best:\n",
    "            best_idx = idx\n",
    "            best_meaning = meaning\n",
    "            best = d\n",
    "    return best_idx, best_meaning\n",
    "\n",
    "def handle_special_char(text): #handling #\n",
    "    newtext = []\n",
    "    for token in text.split():\n",
    "        if \"#\" in token:\n",
    "            token = token.split(\"#\")[0]\n",
    "        newtext.append(token)\n",
    "    return \" \".join(newtext)\n",
    "\n",
    "def count_intersections(s1, s2):\n",
    "    tokens1, tokens2 = set(s1.split()), set(s2.split())\n",
    "    return len(tokens1.intersection(tokens2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "6b14f386",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a9b2baf7d8d472ca87310d9a457e221",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8720.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.756651376146789\n"
     ]
    }
   ],
   "source": [
    "oofs = []\n",
    "for i, row in tqdm(df_train.iterrows(), total=df_train.shape[0]):\n",
    "    text = preprocess(row.text)\n",
    "    tokens = text.split()\n",
    "    \n",
    "    # \"query_token#00007\" => get query_token\n",
    "    query_tokens = [tok for tok in tokens if row.text_id[1:] in tok]\n",
    "    assert len(query_tokens) == 1\n",
    "    query_token = query_tokens[0].split(\"#\")[0]\n",
    "    if query_token.strip() == \"\":\n",
    "        raise Exception(text, row.text, \"\\n\", row)\n",
    "    \n",
    "    # find closest synset\n",
    "    best_syn, query_score = find_closest(query_token)\n",
    "    \n",
    "    # find closest meaning\n",
    "    best_idx, best_meaning = find_closest_meaning(text, best_syn)\n",
    "    \n",
    "    oofs.append((text, query_token, best_syn, query_score, best_idx, best_meaning))\n",
    "\n",
    "df_oof = pd.DataFrame(oofs, columns=[\"text\",\"query\", \"pred_synset\", \"pred_score\", \"pred_synset_id\", \"pred_meaning\"])\n",
    "print(accuracy_score(df_train.synset_id.values, df_oof.pred_synset_id.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "id": "dc3ff344",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>synset_id</th>\n",
       "      <th>synset_word</th>\n",
       "      <th>query</th>\n",
       "      <th>pred_synset</th>\n",
       "      <th>pred_score</th>\n",
       "      <th>pred_synset_id</th>\n",
       "      <th>pred_meaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t0000000001</td>\n",
       "      <td>оны долдугаар сарын#0000000001 13.</td>\n",
       "      <td>43</td>\n",
       "      <td>сар</td>\n",
       "      <td>сарын</td>\n",
       "      <td>сар</td>\n",
       "      <td>1.0</td>\n",
       "      <td>43</td>\n",
       "      <td>он цаг тоолол</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text_id                                 text  synset_id synset_word  \\\n",
       "0  t0000000001   оны долдугаар сарын#0000000001 13.         43         сар   \n",
       "\n",
       "   query pred_synset  pred_score  pred_synset_id   pred_meaning  \n",
       "0  сарын         сар         1.0              43  он цаг тоолол  "
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_oof = pd.concat([df_train, df_oof.drop(\"text\", axis=1)], 1)\n",
    "df_oof.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "id": "2dc62efc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9997706422018349"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_oof.query(\"synset_word==pred_synset\").shape[0] / df_oof.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "id": "9dc3b9ec",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>synset_id</th>\n",
       "      <th>synset_word</th>\n",
       "      <th>query</th>\n",
       "      <th>pred_synset</th>\n",
       "      <th>pred_score</th>\n",
       "      <th>pred_synset_id</th>\n",
       "      <th>pred_meaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>t0000000816</td>\n",
       "      <td>Гэвч миний багын танил Гүрбазар зүгээр л шалбааг гатлах гэж байгаа юм шиг сарнагар хамраа сарталзуулан, нүдээ онийлгон хууч хөөрч, цуваа зургаан дугуйт#0000000816 хүчирхэг &lt;&lt;ЗИЛ&gt;&gt;-ийнхээ жолоог хэнэггүй мушгин Онон руу орж явчихав</td>\n",
       "      <td>53</td>\n",
       "      <td>тэрэг</td>\n",
       "      <td>дугуйт</td>\n",
       "      <td>баг</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "      <td>тодорхой ажил үүргийг хамтран биелүүлэхээр нэг зорилгоор нэгдсэн бүлэг хүн\\nорон нутгийн засаг захиргааны хамгийн бага нэгж: багийн дарга</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3590</th>\n",
       "      <td>t0000003591</td>\n",
       "      <td>Гүнтэв гуай нэг юм гагнан, төмрийн ис, тортгонд дарагдан, хар дээрээ улам хар болчихсон юм шиг, гэвч хийн гагнуурын үе үе сацрах гэрэлд нүүр#0000003591 нь гялалзан, хөлс нь арчих завгүйг мэдсэн юм шиг хоёр хацры нь даган урсах ажээ.</td>\n",
       "      <td>56</td>\n",
       "      <td>хий</td>\n",
       "      <td>нүүр</td>\n",
       "      <td>нүүр</td>\n",
       "      <td>1.0</td>\n",
       "      <td>39</td>\n",
       "      <td>нүүр; царай</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          text_id  \\\n",
       "815   t0000000816   \n",
       "3590  t0000003591   \n",
       "\n",
       "                                                                                                                                                                                                                                          text  \\\n",
       "815     Гэвч миний багын танил Гүрбазар зүгээр л шалбааг гатлах гэж байгаа юм шиг сарнагар хамраа сарталзуулан, нүдээ онийлгон хууч хөөрч, цуваа зургаан дугуйт#0000000816 хүчирхэг <<ЗИЛ>>-ийнхээ жолоог хэнэггүй мушгин Онон руу орж явчихав   \n",
       "3590  Гүнтэв гуай нэг юм гагнан, төмрийн ис, тортгонд дарагдан, хар дээрээ улам хар болчихсон юм шиг, гэвч хийн гагнуурын үе үе сацрах гэрэлд нүүр#0000003591 нь гялалзан, хөлс нь арчих завгүйг мэдсэн юм шиг хоёр хацры нь даган урсах ажээ.   \n",
       "\n",
       "      synset_id synset_word   query pred_synset  pred_score  pred_synset_id  \\\n",
       "815          53       тэрэг  дугуйт         баг         0.0               7   \n",
       "3590         56         хий    нүүр        нүүр         1.0              39   \n",
       "\n",
       "                                                                                                                                   pred_meaning  \n",
       "815   тодорхой ажил үүргийг хамтран биелүүлэхээр нэг зорилгоор нэгдсэн бүлэг хүн\\nорон нутгийн засаг захиргааны хамгийн бага нэгж: багийн дарга  \n",
       "3590                                                                                                                                нүүр; царай  "
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_oof.query(\"synset_word!=pred_synset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "d3fea673",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aacd2c5f4214af5aaa3277e7972d720",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17028.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "for index, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):\n",
    "    text = preprocess_test(row.text)\n",
    "    tokens = text.split()\n",
    "    \n",
    "    # \"query_token#00007\" => get query_token\n",
    "    query_tokens = [tok for tok in tokens if row.text_id[1:] in tok]\n",
    "    assert len(query_tokens) == 1\n",
    "    query_token = query_tokens[0].split(\"#\")[0]\n",
    "    if query_token.strip() == \"\":\n",
    "        raise Exception(text, row.text, \"\\n\", row)\n",
    "    \n",
    "    # find closest synset\n",
    "    best_syn, query_score = find_closest(query_token)\n",
    "    \n",
    "    # find closest meaning\n",
    "    best_idx, best_meaning = find_closest_meaning(text, best_syn)\n",
    "    predictions.append((text, query_token, best_syn, query_score, best_idx, best_meaning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "746e4436",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17028, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>query_word</th>\n",
       "      <th>synset_word</th>\n",
       "      <th>score</th>\n",
       "      <th>synset_id</th>\n",
       "      <th>meaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t0000008721</td>\n",
       "      <td>Энэ хэсэгт  оны  дүгээр сарын#0000008721 2ны өдрийн хуулиар өөрчлөлт орсон</td>\n",
       "      <td>сарын</td>\n",
       "      <td>сар</td>\n",
       "      <td>1.0</td>\n",
       "      <td>43</td>\n",
       "      <td>он цаг тоолол</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>t0000008722</td>\n",
       "      <td>Хадлангийн үеэр зарлах уралдааны болзлын өвс бэлтгэх ажлын зохистой хугацаа, өвсний чанартай шууд холбож, ажиллагчдын хөдөлмөрийн идэвхийг өрнүүлэхүйцээр хийж#0000008722 түүнийгээ хадланч бүрт одооноос сайтар ойлгуулах нь зүйтэй</td>\n",
       "      <td>хийж</td>\n",
       "      <td>хий</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54</td>\n",
       "      <td>ажил хийх, үйлдэх, хөдөлмөрлөх, ажиллах</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text_id  \\\n",
       "0  t0000008721   \n",
       "1  t0000008722   \n",
       "\n",
       "                                                                                                                                                                                                                                   text  \\\n",
       "0                                                                                                                                                            Энэ хэсэгт  оны  дүгээр сарын#0000008721 2ны өдрийн хуулиар өөрчлөлт орсон   \n",
       "1  Хадлангийн үеэр зарлах уралдааны болзлын өвс бэлтгэх ажлын зохистой хугацаа, өвсний чанартай шууд холбож, ажиллагчдын хөдөлмөрийн идэвхийг өрнүүлэхүйцээр хийж#0000008722 түүнийгээ хадланч бүрт одооноос сайтар ойлгуулах нь зүйтэй   \n",
       "\n",
       "  query_word synset_word  score  synset_id  \\\n",
       "0      сарын         сар    1.0         43   \n",
       "1       хийж         хий    1.0         54   \n",
       "\n",
       "                                   meaning  \n",
       "0                            он цаг тоолол  \n",
       "1  ажил хийх, үйлдэх, хөдөлмөрлөх, ажиллах  "
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_predictions = pd.DataFrame(predictions, columns=[\"text\",\"query_word\", \"synset_word\", \"score\", \"synset_id\", \"meaning\"])\n",
    "df_predictions = pd.concat([df_test[[\"text_id\"]], df_predictions], axis=1)\n",
    "df_test = pd.concat([df_test, df_predictions[[\"synset_word\"]].rename({\"synset_word\":\"pred_synset_word\"})], axis=1)\n",
    "print(df_predictions.shape)\n",
    "df_predictions.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "id": "f7013490",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>query_word</th>\n",
       "      <th>synset_word</th>\n",
       "      <th>score</th>\n",
       "      <th>synset_id</th>\n",
       "      <th>meaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1499</th>\n",
       "      <td>t0000010220</td>\n",
       "      <td>Машин залуураа эргүүлэхэд жаран хөлт хорхой шиг хойд урд, дунд бүх дугуй#0000010220 нь мурийж хонгилоор могой аятай мурилзан явдаг</td>\n",
       "      <td>дугуй</td>\n",
       "      <td>баг</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7</td>\n",
       "      <td>тодорхой ажил үүргийг хамтран биелүүлэхээр нэг зорилгоор нэгдсэн бүлэг хүн\\nорон нутгийн засаг захиргааны хамгийн бага нэгж: багийн дарга</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4083</th>\n",
       "      <td>t0000012804</td>\n",
       "      <td>Үнэндээ уул уурхайгаас орж ирсэн мөнгийг#0000012804 хүндээ хөрөнгө оруулалт хийвэл  арван жилийн дараа өнөөдөр зарсан мөнгөнөөс арав дахин их мөнгийг Монголд босгож, гаргаж ирэх чадалтай шүү дээ</td>\n",
       "      <td>мөнгийг</td>\n",
       "      <td>мөнгө</td>\n",
       "      <td>0.8</td>\n",
       "      <td>33</td>\n",
       "      <td>мөнгөний тоо; хэмжээ; мөнгө</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4086</th>\n",
       "      <td>t0000012807</td>\n",
       "      <td>Үнэндээ уул уурхайгаас орж ирсэн мөнгийг хүндээ хөрөнгө оруулалт хийвэл арван жилийн дараа өнөөдөр зарсан мөнгөнөөс арав дахин их мөнгийг#0000012807 Монголд босгож, гаргаж ирэх чадалтай шүү дээ</td>\n",
       "      <td>мөнгийг</td>\n",
       "      <td>мөнгө</td>\n",
       "      <td>0.8</td>\n",
       "      <td>33</td>\n",
       "      <td>мөнгөний тоо; хэмжээ; мөнгө</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          text_id  \\\n",
       "1499  t0000010220   \n",
       "4083  t0000012804   \n",
       "4086  t0000012807   \n",
       "\n",
       "                                                                                                                                                                                                    text  \\\n",
       "1499                                                                  Машин залуураа эргүүлэхэд жаран хөлт хорхой шиг хойд урд, дунд бүх дугуй#0000010220 нь мурийж хонгилоор могой аятай мурилзан явдаг   \n",
       "4083  Үнэндээ уул уурхайгаас орж ирсэн мөнгийг#0000012804 хүндээ хөрөнгө оруулалт хийвэл  арван жилийн дараа өнөөдөр зарсан мөнгөнөөс арав дахин их мөнгийг Монголд босгож, гаргаж ирэх чадалтай шүү дээ   \n",
       "4086   Үнэндээ уул уурхайгаас орж ирсэн мөнгийг хүндээ хөрөнгө оруулалт хийвэл арван жилийн дараа өнөөдөр зарсан мөнгөнөөс арав дахин их мөнгийг#0000012807 Монголд босгож, гаргаж ирэх чадалтай шүү дээ   \n",
       "\n",
       "     query_word synset_word  score  synset_id  \\\n",
       "1499      дугуй         баг    0.0          7   \n",
       "4083    мөнгийг       мөнгө    0.8         33   \n",
       "4086    мөнгийг       мөнгө    0.8         33   \n",
       "\n",
       "                                                                                                                                        meaning  \n",
       "1499  тодорхой ажил үүргийг хамтран биелүүлэхээр нэг зорилгоор нэгдсэн бүлэг хүн\\nорон нутгийн засаг захиргааны хамгийн бага нэгж: багийн дарга  \n",
       "4083                                                                                                                мөнгөний тоо; хэмжээ; мөнгө  \n",
       "4086                                                                                                                мөнгөний тоо; хэмжээ; мөнгө  "
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_predictions.query(\"score < 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "8c69bbbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>query_word</th>\n",
       "      <th>synset_word</th>\n",
       "      <th>score</th>\n",
       "      <th>synset_id</th>\n",
       "      <th>meaning</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t0000008721</td>\n",
       "      <td>Энэ хэсэгт  оны  дүгээр сарын#0000008721 2ны өдрийн хуулиар өөрчлөлт орсон</td>\n",
       "      <td>сарын</td>\n",
       "      <td>сар</td>\n",
       "      <td>1.0</td>\n",
       "      <td>43</td>\n",
       "      <td>он цаг тоолол</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text_id  \\\n",
       "0  t0000008721   \n",
       "\n",
       "                                                                         text  \\\n",
       "0  Энэ хэсэгт  оны  дүгээр сарын#0000008721 2ны өдрийн хуулиар өөрчлөлт орсон   \n",
       "\n",
       "  query_word synset_word  score  synset_id        meaning  \n",
       "0      сарын         сар    1.0         43  он цаг тоолол  "
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_predictions[[\"text_id\",\"text\",\"query_word\",\"synset_word\",\"score\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "a4de61ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>synset_id</th>\n",
       "      <th>synset_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t0000000001</td>\n",
       "      <td>оны долдугаар сарын#0000000001 13.</td>\n",
       "      <td>43</td>\n",
       "      <td>сар</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text_id                                 text  synset_id synset_word\n",
       "0  t0000000001   оны долдугаар сарын#0000000001 13.         43         сар"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "79343466",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv(\"data/preprocessed/train.csv\", index=False)\n",
    "df_predictions[\n",
    "    [\"text_id\",\"text\",\"query_word\",\"synset_word\",\"score\"]\n",
    "].to_csv(\"data/preprocessed/test.csv\", index=False)\n",
    "df_synset_meaning.to_csv(\"data/preprocessed/synset_meaning.csv\", index=False)\n",
    "df_submission.to_csv(\"data/preprocessed/submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ca0f006",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80d48208",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "ebc4d582",
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "id": "4ac84734",
   "metadata": {},
   "outputs": [],
   "source": [
    "process = subprocess.run([\n",
    "    \"kaggle\",\n",
    "    \"competitions\",\n",
    "    \"submit\",\n",
    "    \"-c\",\n",
    "    \"muis-challenge\",\n",
    "    \"-f\",\n",
    "    \"submission.csv\",\n",
    "    \"-m\",\n",
    "    \"Message\"\n",
    "], capture_output=True)\n",
    "\n",
    "kaggle competitions submit -c muis-challenge -f submission.csv -m \"Message\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "eaef5738",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'ref                                                        title                                          size  lastUpdated          downloadCount  voteCount  usabilityRating  \\n---------------------------------------------------------  --------------------------------------------  -----  -------------------  -------------  ---------  ---------------  \\nbayartsogtya/secret-kaggle-credential                      [SECRET] Kaggle Credential                     229B  2021-08-11 18:17:53              0          0  0.25             \\nbayartsogtya/bayartsogt-rob-l-p-cl-ah-orig-s1000           bayartsogt-rob-l-p-cl-ah-orig-s1000             6GB  2021-07-31 21:59:41              0          0  0.3529412        \\nbayartsogtya/ub-weather-2015-2020                          ulaanbaatar-weather-2015-2020                 988KB  2020-12-08 21:29:49             22          2  0.7058824        \\nbayartsogtya/mlub-roberta-base-v0                          mlub-roberta-base-v0                            3GB  2021-08-21 22:08:20              0          0  0.23529412       \\nbayartsogtya/cl-roberta-base                               CL RoBerta Base 0.47                            2GB  2021-07-23 15:44:12              0          0  0.25             \\nbayartsogtya/kaggle-commonlit-exp-no0                      kaggle commonlit exp no0                        4GB  2021-07-24 17:35:18              0          0  0.3125           \\nbayartsogtya/mlub-embed-mongolian-roberta-base             mlub embed mongolian roberta base              70MB  2021-08-19 17:50:47              0          0  0.0625           \\nbayartsogtya/roberta-large-ah-orig-s43                     roberta-large-ah-orig-s43                       6GB  2021-07-27 13:54:29              0          0  0.375            \\nbayartsogtya/mlub-mongolian-car-plate-prediction-v1        mlub mongolian car plate prediction v1         34KB  2020-10-22 17:31:16             68          0  0.29411766       \\nbayartsogtya/hf-autonlp-roberta-base                       HF AutoNLP Roberta Base                       853MB  2021-07-15 17:38:52              0          0  0.29411766       \\nbayartsogtya/commonlit-backtranslated                      CommonLit Backtranslated                        9MB  2021-07-26 19:21:50              0          0  0.11764706       \\nbayartsogtya/my-audio-batch0                               my-audio-batch0                                 2GB  2021-03-29 20:47:04              3          5  0.375            \\nbayartsogtya/ulaanbal-videos-v0                            \\xd0\\xa3\\xd0\\xbb\\xd0\\xb0\\xd0\\xb0\\xd0\\xbd\\xd0\\xb1\\xd0\\xb0\\xd0\\xbb \\xd0\\xb0\\xd1\\x83\\xd0\\xb4\\xd0\\xb8\\xd0\\xbe \\xd1\\x84\\xd0\\xb0\\xd0\\xb9\\xd0\\xbb\\xd1\\x83\\xd1\\x83\\xd0\\xb4 v0 by 03/28/2021        41GB  2021-03-29 02:03:21              1          1  0.25             \\nbayartsogtya/idree-podcast-v0                              Ideree Podcast v0 by 04 April, 2021             8GB  2021-04-08 04:02:43              3          0  0.23529412       \\nbayartsogtya/indoor-navigation-and-location-wifi-features  indoor-navigation-and-location-wifi-features  123MB  2021-02-11 17:31:55              5          0  0.3529412        \\nbayartsogtya/mongolian-speech-commands                     Mongolian Speech Commands                      40MB  2021-03-11 15:50:01              1          0  0.3125           \\nbayartsogtya/indoor-for-subm                               [indoor] for subm                             660KB  2021-03-18 03:52:41              0          0  0.11764706       \\nbayartsogtya/ub-station-elevation                          UB Station Elevation                           571B  2020-12-02 00:00:39              0          0  0.23529412       \\nbayartsogtya/pinknoise                                     Pink Noise SR 48k                               2GB  2021-01-17 06:10:11             21          1  0.1875           \\nbayartsogtya/rfcx-submissions                              RFCX Submissions                                5MB  2021-02-13 21:49:20              0          0  0.11764706       \\n'\n"
     ]
    }
   ],
   "source": [
    "print(process.stdout)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}