rks_data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2ef2700a-0ab5-486e-a261-5f369c2d3aab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "42ef1d2a-9c23-4f20-8ab4-de63716b2ddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('roskomvsoboda_chlb_merge.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "115c0731-36ab-49f1-bc21-6be1a20edab7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tweak_decnum_scrapping(df):\n",
    "    df_ = (df\n",
    "           .loc[:, ['record_id', 'decisionNumber', 'decisionDate', \n",
    "                    'authority_id', 'authority_name', 'status',\n",
    "                    'domains', 'urls']]\n",
    "          )\n",
    "    return df_\n",
    "\n",
    "df = tweak_decnum_scrapping(df)#.to_csv('roskomvsoboda_chlb_scrapping.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d91cadec-aa0f-449c-86c1-a2c761ab6942",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_id</th>\n",
       "      <th>decisionNumber</th>\n",
       "      <th>decisionDate</th>\n",
       "      <th>authority_id</th>\n",
       "      <th>authority_name</th>\n",
       "      <th>status</th>\n",
       "      <th>domains</th>\n",
       "      <th>urls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2904472</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2021-01-15</td>\n",
       "      <td>103</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['prostitutkikurganagirls.com']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2924065</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2021-01-15</td>\n",
       "      <td>103</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['prostitutkikurganabest.com']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2924066</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2021-01-15</td>\n",
       "      <td>103</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['sex-portal.org']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2961133</td>\n",
       "      <td>2а-14/20</td>\n",
       "      <td>2021-01-19</td>\n",
       "      <td>714</td>\n",
       "      <td>Нязепетровский районный суд - Челябинская область</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['xx-74.ru']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2966748</td>\n",
       "      <td>2а-12/2021</td>\n",
       "      <td>2021-01-19</td>\n",
       "      <td>714</td>\n",
       "      <td>Нязепетровский районный суд - Челябинская область</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['chelyabinsk.diplomnarukah.com']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2125</th>\n",
       "      <td>5496088</td>\n",
       "      <td>2а-1212/2022</td>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>1836</td>\n",
       "      <td>Карталинский городской суд - Челябинская область</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['socuteporn.com']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2126</th>\n",
       "      <td>5496540</td>\n",
       "      <td>2а-1210/2022</td>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>1836</td>\n",
       "      <td>Карталинский городской суд - Челябинская область</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['www.ruseks.top']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2127</th>\n",
       "      <td>5524205</td>\n",
       "      <td>2а-8096/2022</td>\n",
       "      <td>2022-12-14</td>\n",
       "      <td>66</td>\n",
       "      <td>Курчатовский районный суд г. Челябинска - Челя...</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['med.spravki-za-chas.top']</td>\n",
       "      <td>['https://med.spravki-za-chas.top/product/medi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2128</th>\n",
       "      <td>5529744</td>\n",
       "      <td>2а-8097/2022</td>\n",
       "      <td>2022-12-14</td>\n",
       "      <td>66</td>\n",
       "      <td>Курчатовский районный суд г. Челябинска - Челя...</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['spravkak-magazine.com']</td>\n",
       "      <td>['https://spravkak-magazine.com/spravka-na-oru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2129</th>\n",
       "      <td>5547912</td>\n",
       "      <td>2а-2772/2022</td>\n",
       "      <td>2022-12-30</td>\n",
       "      <td>1514</td>\n",
       "      <td>Кыштымский городской суд - Челябинская область</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['g.intimsity.com']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2130 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      record_id decisionNumber decisionDate  authority_id  \\\n",
       "0       2904472            б/н   2021-01-15           103   \n",
       "1       2924065            б/н   2021-01-15           103   \n",
       "2       2924066            б/н   2021-01-15           103   \n",
       "3       2961133       2а-14/20   2021-01-19           714   \n",
       "4       2966748     2а-12/2021   2021-01-19           714   \n",
       "...         ...            ...          ...           ...   \n",
       "2125    5496088   2а-1212/2022   2022-12-29          1836   \n",
       "2126    5496540   2а-1210/2022   2022-12-29          1836   \n",
       "2127    5524205   2а-8096/2022   2022-12-14            66   \n",
       "2128    5529744   2а-8097/2022   2022-12-14            66   \n",
       "2129    5547912   2а-2772/2022   2022-12-30          1514   \n",
       "\n",
       "                                         authority_name   status  \\\n",
       "0     Центральный районный суд г. Челябинска - Челяб...  blocked   \n",
       "1     Центральный районный суд г. Челябинска - Челяб...  blocked   \n",
       "2     Центральный районный суд г. Челябинска - Челяб...  blocked   \n",
       "3     Нязепетровский районный суд - Челябинская область  blocked   \n",
       "4     Нязепетровский районный суд - Челябинская область  blocked   \n",
       "...                                                 ...      ...   \n",
       "2125   Карталинский городской суд - Челябинская область  blocked   \n",
       "2126   Карталинский городской суд - Челябинская область  blocked   \n",
       "2127  Курчатовский районный суд г. Челябинска - Челя...  blocked   \n",
       "2128  Курчатовский районный суд г. Челябинска - Челя...  blocked   \n",
       "2129     Кыштымский городской суд - Челябинская область  blocked   \n",
       "\n",
       "                                domains  \\\n",
       "0       ['prostitutkikurganagirls.com']   \n",
       "1        ['prostitutkikurganabest.com']   \n",
       "2                    ['sex-portal.org']   \n",
       "3                          ['xx-74.ru']   \n",
       "4     ['chelyabinsk.diplomnarukah.com']   \n",
       "...                                 ...   \n",
       "2125                 ['socuteporn.com']   \n",
       "2126                 ['www.ruseks.top']   \n",
       "2127        ['med.spravki-za-chas.top']   \n",
       "2128          ['spravkak-magazine.com']   \n",
       "2129                ['g.intimsity.com']   \n",
       "\n",
       "                                                   urls  \n",
       "0                                                    []  \n",
       "1                                                    []  \n",
       "2                                                    []  \n",
       "3                                                    []  \n",
       "4                                                    []  \n",
       "...                                                 ...  \n",
       "2125                                                 []  \n",
       "2126                                                 []  \n",
       "2127  ['https://med.spravki-za-chas.top/product/medi...  \n",
       "2128  ['https://spravkak-magazine.com/spravka-na-oru...  \n",
       "2129                                                 []  \n",
       "\n",
       "[2130 rows x 8 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2021 = pd.read_csv('roskomvsoboda_chlb_scrapping_2021.csv')\n",
    "\n",
    "df_con = pd.concat([df_2021, df], ignore_index=True)\n",
    "df_con"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "56340d04-97df-402c-8f2e-483b42df5327",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "б/н                           35\n",
       "2а-3820/2022                  13\n",
       "2a-794/2022                   11\n",
       "2а-891/2022                    7\n",
       "2а-163/2021                    7\n",
       "                              ..\n",
       "2a-2689/2022                   1\n",
       "2a-2688/2022                   1\n",
       "2a-785/2022 (2а-5029/2021)     1\n",
       "2a-2612/2022                   1\n",
       "2а-8097/2022                   1\n",
       "Name: decisionNumber, Length: 1592, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_con.decisionNumber.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f35c2aa0-87b8-4062-afff-151ce897deed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_id</th>\n",
       "      <th>decisionNumber</th>\n",
       "      <th>decisionDate</th>\n",
       "      <th>authority_id</th>\n",
       "      <th>authority_name</th>\n",
       "      <th>status</th>\n",
       "      <th>domains</th>\n",
       "      <th>urls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>4119250</td>\n",
       "      <td>2a-2689/2022</td>\n",
       "      <td>2022-02-17</td>\n",
       "      <td>103</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "      <td>blocked</td>\n",
       "      <td>['cosplay-porn.me']</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     record_id decisionNumber decisionDate  authority_id  \\\n",
       "698    4119250   2a-2689/2022   2022-02-17           103   \n",
       "\n",
       "                                        authority_name   status  \\\n",
       "698  Центральный районный суд г. Челябинска - Челяб...  blocked   \n",
       "\n",
       "                 domains urls  \n",
       "698  ['cosplay-porn.me']   []  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_con.loc[df_con.decisionNumber == '2a-2689/2022']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f12597c4-b4c7-4ed0-be56-691cc61fac06",
   "metadata": {},
   "source": [
    "Extract domain name. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3e7e3d66-b0f0-46fc-8f1a-76c5483d8d7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generalize_topn(ser, n=10, other='Other'):\n",
    "    topn = ser.value_counts().index[:n]\n",
    "    if isinstance(ser.dtype, pd.CategoricalDtype):\n",
    "        ser = ser.cat.set_categories( \n",
    "            topn.set_categories(list(topn)+[other]))\n",
    "    return ser.where(ser.isin(topn), other)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "3e811c4c-fdf5-4101-b3b3-2f7f96236eea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_id</th>\n",
       "      <th>decisionNumber</th>\n",
       "      <th>decisionDate</th>\n",
       "      <th>decision_month</th>\n",
       "      <th>decision_year</th>\n",
       "      <th>domains</th>\n",
       "      <th>domain_register</th>\n",
       "      <th>authority_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2904472</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2021-01-15</td>\n",
       "      <td>2021-01</td>\n",
       "      <td>2021</td>\n",
       "      <td>['prostitutkikurganagirls.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2924065</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2021-01-15</td>\n",
       "      <td>2021-01</td>\n",
       "      <td>2021</td>\n",
       "      <td>['prostitutkikurganabest.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2924066</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2021-01-15</td>\n",
       "      <td>2021-01</td>\n",
       "      <td>2021</td>\n",
       "      <td>['sex-portal.org']</td>\n",
       "      <td>.org</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2961133</td>\n",
       "      <td>2а-14/20</td>\n",
       "      <td>2021-01-19</td>\n",
       "      <td>2021-01</td>\n",
       "      <td>2021</td>\n",
       "      <td>['xx-74.ru']</td>\n",
       "      <td>.ru</td>\n",
       "      <td>Нязепетровский районный суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2966748</td>\n",
       "      <td>2а-12/2021</td>\n",
       "      <td>2021-01-19</td>\n",
       "      <td>2021-01</td>\n",
       "      <td>2021</td>\n",
       "      <td>['chelyabinsk.diplomnarukah.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Нязепетровский районный суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2125</th>\n",
       "      <td>5496088</td>\n",
       "      <td>2а-1212/2022</td>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['socuteporn.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Карталинский городской суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2126</th>\n",
       "      <td>5496540</td>\n",
       "      <td>2а-1210/2022</td>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['www.ruseks.top']</td>\n",
       "      <td>.top</td>\n",
       "      <td>Карталинский городской суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2127</th>\n",
       "      <td>5524205</td>\n",
       "      <td>2а-8096/2022</td>\n",
       "      <td>2022-12-14</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['med.spravki-za-chas.top']</td>\n",
       "      <td>.top</td>\n",
       "      <td>Курчатовский районный суд г. Челябинска - Челя...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2128</th>\n",
       "      <td>5529744</td>\n",
       "      <td>2а-8097/2022</td>\n",
       "      <td>2022-12-14</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['spravkak-magazine.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Курчатовский районный суд г. Челябинска - Челя...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2129</th>\n",
       "      <td>5547912</td>\n",
       "      <td>2а-2772/2022</td>\n",
       "      <td>2022-12-30</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['g.intimsity.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Кыштымский городской суд - Челябинская область</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2130 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      record_id decisionNumber decisionDate decision_month decision_year  \\\n",
       "0       2904472            б/н   2021-01-15        2021-01          2021   \n",
       "1       2924065            б/н   2021-01-15        2021-01          2021   \n",
       "2       2924066            б/н   2021-01-15        2021-01          2021   \n",
       "3       2961133       2а-14/20   2021-01-19        2021-01          2021   \n",
       "4       2966748     2а-12/2021   2021-01-19        2021-01          2021   \n",
       "...         ...            ...          ...            ...           ...   \n",
       "2125    5496088   2а-1212/2022   2022-12-29        2022-12          2022   \n",
       "2126    5496540   2а-1210/2022   2022-12-29        2022-12          2022   \n",
       "2127    5524205   2а-8096/2022   2022-12-14        2022-12          2022   \n",
       "2128    5529744   2а-8097/2022   2022-12-14        2022-12          2022   \n",
       "2129    5547912   2а-2772/2022   2022-12-30        2022-12          2022   \n",
       "\n",
       "                                domains domain_register  \\\n",
       "0       ['prostitutkikurganagirls.com']            .com   \n",
       "1        ['prostitutkikurganabest.com']            .com   \n",
       "2                    ['sex-portal.org']            .org   \n",
       "3                          ['xx-74.ru']             .ru   \n",
       "4     ['chelyabinsk.diplomnarukah.com']            .com   \n",
       "...                                 ...             ...   \n",
       "2125                 ['socuteporn.com']            .com   \n",
       "2126                 ['www.ruseks.top']            .top   \n",
       "2127        ['med.spravki-za-chas.top']            .top   \n",
       "2128          ['spravkak-magazine.com']            .com   \n",
       "2129                ['g.intimsity.com']            .com   \n",
       "\n",
       "                                         authority_name  \n",
       "0     Центральный районный суд г. Челябинска - Челяб...  \n",
       "1     Центральный районный суд г. Челябинска - Челяб...  \n",
       "2     Центральный районный суд г. Челябинска - Челяб...  \n",
       "3     Нязепетровский районный суд - Челябинская область  \n",
       "4     Нязепетровский районный суд - Челябинская область  \n",
       "...                                                 ...  \n",
       "2125   Карталинский городской суд - Челябинская область  \n",
       "2126   Карталинский городской суд - Челябинская область  \n",
       "2127  Курчатовский районный суд г. Челябинска - Челя...  \n",
       "2128  Курчатовский районный суд г. Челябинска - Челя...  \n",
       "2129     Кыштымский городской суд - Челябинская область  \n",
       "\n",
       "[2130 rows x 8 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dom_analysis = (df_con\n",
    "                 .assign(domain_register=lambda df_: df_.domains.str.extract(r\"^\\['\\S+(\\.\\w{1,6})'\\]$\", expand=False),\n",
    "                         decision_month=lambda df_: df_.decisionDate.str[:7],\n",
    "                         decision_year=lambda df_: df_.decisionDate.str[:4])\n",
    "                 .loc[:, ['record_id', 'decisionNumber', 'decisionDate', 'decision_month', \n",
    "                          'decision_year', 'domains', 'domain_register', 'authority_name']]\n",
    "                )\n",
    "\n",
    "df_dom_analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5bed7b1-4b58-4c7d-9c39-7f6d7e83151b",
   "metadata": {},
   "source": [
    "Make simple substring-based filter to remove the most obvious non-fitting cases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "663dd025-8e2b-4576-afad-b20cc2d430ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_id</th>\n",
       "      <th>decisionNumber</th>\n",
       "      <th>decisionDate</th>\n",
       "      <th>decision_month</th>\n",
       "      <th>decision_year</th>\n",
       "      <th>domains</th>\n",
       "      <th>domain_register</th>\n",
       "      <th>authority_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>3049955</td>\n",
       "      <td>2а-418-2021</td>\n",
       "      <td>2021-02-17</td>\n",
       "      <td>2021-02</td>\n",
       "      <td>2021</td>\n",
       "      <td>['thomas-stege.de']</td>\n",
       "      <td>.de</td>\n",
       "      <td>Саткинский городской суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>4343144</td>\n",
       "      <td>2а-1289/2021</td>\n",
       "      <td>2021-02-19</td>\n",
       "      <td>2021-02</td>\n",
       "      <td>2021</td>\n",
       "      <td>['admiralwin.fun']</td>\n",
       "      <td>.fun</td>\n",
       "      <td>Курчатовский районный суд г. Челябинска - Челя...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>3289057</td>\n",
       "      <td>2а-629/2021</td>\n",
       "      <td>2021-02-20</td>\n",
       "      <td>2021-02</td>\n",
       "      <td>2021</td>\n",
       "      <td>['piholpua.ds216.ru']</td>\n",
       "      <td>.ru</td>\n",
       "      <td>Правобережный районный суд г. Магнитогорска - ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>3288196</td>\n",
       "      <td>2а-631/2021</td>\n",
       "      <td>2021-02-20</td>\n",
       "      <td>2021-02</td>\n",
       "      <td>2021</td>\n",
       "      <td>['kesalon.victoria-rossi.ru']</td>\n",
       "      <td>.ru</td>\n",
       "      <td>Правобережный районный суд г. Магнитогорска - ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>3139653</td>\n",
       "      <td>2а-768/2021</td>\n",
       "      <td>2021-02-24</td>\n",
       "      <td>2021-02</td>\n",
       "      <td>2021</td>\n",
       "      <td>['playfortunagl12.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Металлургический районный суд г. Челябинска - ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1904</th>\n",
       "      <td>5242102</td>\n",
       "      <td>2а-815/2022 ~ М-812/2022</td>\n",
       "      <td>2022-12-23</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['snlines.shop']</td>\n",
       "      <td>.shop</td>\n",
       "      <td>Нагайбакский районный суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>5278489</td>\n",
       "      <td>2а-729/2022 ~ М-698/2022</td>\n",
       "      <td>2022-12-26</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['dokhumentn-ohrana.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Усть-Катавский городской суд - Челябинская обл...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2103</th>\n",
       "      <td>5455005</td>\n",
       "      <td>2а-2622/2022</td>\n",
       "      <td>2022-12-28</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['ozersk.soos.su']</td>\n",
       "      <td>.su</td>\n",
       "      <td>Кыштымский городской суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2111</th>\n",
       "      <td>5459899</td>\n",
       "      <td>2а-1217/2022</td>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['chelyabinsk.extern-dip.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Карталинский городской суд - Челябинская область</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>5293333</td>\n",
       "      <td>б/н</td>\n",
       "      <td>2022-12-30</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>2022</td>\n",
       "      <td>['one.indi-samara11.com']</td>\n",
       "      <td>.com</td>\n",
       "      <td>Центральный районный суд г. Челябинска - Челяб...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>641 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      record_id            decisionNumber decisionDate decision_month  \\\n",
       "21      3049955               2а-418-2021   2021-02-17        2021-02   \n",
       "616     4343144              2а-1289/2021   2021-02-19        2021-02   \n",
       "117     3289057               2а-629/2021   2021-02-20        2021-02   \n",
       "116     3288196               2а-631/2021   2021-02-20        2021-02   \n",
       "60      3139653               2а-768/2021   2021-02-24        2021-02   \n",
       "...         ...                       ...          ...            ...   \n",
       "1904    5242102  2а-815/2022 ~ М-812/2022   2022-12-23        2022-12   \n",
       "2008    5278489  2а-729/2022 ~ М-698/2022   2022-12-26        2022-12   \n",
       "2103    5455005              2а-2622/2022   2022-12-28        2022-12   \n",
       "2111    5459899              2а-1217/2022   2022-12-29        2022-12   \n",
       "2016    5293333                       б/н   2022-12-30        2022-12   \n",
       "\n",
       "     decision_year                         domains domain_register  \\\n",
       "21            2021             ['thomas-stege.de']             .de   \n",
       "616           2021              ['admiralwin.fun']            .fun   \n",
       "117           2021           ['piholpua.ds216.ru']             .ru   \n",
       "116           2021   ['kesalon.victoria-rossi.ru']             .ru   \n",
       "60            2021         ['playfortunagl12.com']            .com   \n",
       "...            ...                             ...             ...   \n",
       "1904          2022                ['snlines.shop']           .shop   \n",
       "2008          2022       ['dokhumentn-ohrana.com']            .com   \n",
       "2103          2022              ['ozersk.soos.su']             .su   \n",
       "2111          2022  ['chelyabinsk.extern-dip.com']            .com   \n",
       "2016          2022       ['one.indi-samara11.com']            .com   \n",
       "\n",
       "                                         authority_name  \n",
       "21       Саткинский городской суд - Челябинская область  \n",
       "616   Курчатовский районный суд г. Челябинска - Челя...  \n",
       "117   Правобережный районный суд г. Магнитогорска - ...  \n",
       "116   Правобережный районный суд г. Магнитогорска - ...  \n",
       "60    Металлургический районный суд г. Челябинска - ...  \n",
       "...                                                 ...  \n",
       "1904    Нагайбакский районный суд - Челябинская область  \n",
       "2008  Усть-Катавский городской суд - Челябинская обл...  \n",
       "2103     Кыштымский городской суд - Челябинская область  \n",
       "2111   Карталинский городской суд - Челябинская область  \n",
       "2016  Центральный районный суд г. Челябинска - Челяб...  \n",
       "\n",
       "[641 rows x 8 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words = ['audio', 'video', 'sliv', 'kino', 'film', 'tv', 'movie', 'resheba', 'books', 'sex',\n",
    "              'klad', 'nark', 'torrent', 'muz', 'knigi', 'hd', 'serial', 'series', 'tracker',\n",
    "              'bill', 'prostitutk', 'porn', 'intim', 'seks', 'amur', 'feiportal', 'kurtizanki', 'dosug',\n",
    "              'shluh', 'xx', 'murmur', 'diplom', 'prava', 'dublikat', 'casino', 'devki', 'soski', 'parnuha',\n",
    "              'boobs', 'eccemoda', 'oruzhie', 'love', 'tube', 'lady', 'spravk', 'putan', 'damochki', 'doska',\n",
    "              'devochki', 'obnal', 'dokument', 'pvkgb', 'guns', 'document', 'slut', 'prostutytku', 'kiss',\n",
    "              'whore', 'otsos', '4zzy', 'transi', 'sprawka', 'individualka', 'titki', 'pizd', 'siski', 'ebu',\n",
    "              'doc', 'golye', 'escort', 'dildos', 'harlots']\n",
    "\n",
    "(df_dom_analysis\n",
    " .loc[~df_dom_analysis.domains.str.contains(\"|\".join(stop_words), regex=True)]\n",
    " .sort_values('decisionDate')\n",
    ")#.to_csv('chelyabinsk_2021_2022_cases.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfb134c0-8f07-49cf-b1c4-5fb8ec22ea51",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}