get_yandex_ratings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from yandex.get_yandex_ratings import get_items\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from math import sin, cos, sqrt, atan2, radians\n",
    "from tqdm import tqdm\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.markers import MarkerStyle\n",
    "from postal.parser import parse_address\n",
    "from tqdm import tqdm\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from scipy.interpolate import spline\n",
    "from sklearn.cluster import DBSCAN\n",
    "from collections import Counter\n",
    "import json\n",
    "\n",
    "\n",
    "\n",
    "fontsize = 18\n",
    "figsize = (15,10)\n",
    "plot_alpha = 0.3\n",
    "linewidth = 4\n",
    "\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y_true, y_pred):\n",
    "    return sqrt(mean_squared_error(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_to_point(s):\n",
    "    if s is None:\n",
    "        return None\n",
    "    l = s.split(' ')\n",
    "    return {'long': float(l[0]), 'lat': float(l[1])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/train.csv', index_col=0)\n",
    "test = pd.read_csv('data/test.csv', index_col=0)\n",
    "test = test.rename(columns={'atm_group': '0-bank'})\n",
    "train = train.rename(columns={'atm_group': '0-bank'})\n",
    "banks = np.uint(np.sort(test['0-bank'].unique() * 10))\n",
    "\n",
    "with open('missed_coords.json', 'r') as infile:\n",
    "    missed_coords = json.load(infile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "missed_coords = [(str_to_point(item[0]), item[1], item[2], item[3]) for item in missed_coords]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test.rename(columns={'lat': '6-lat', 'long': '6-long'})\n",
    "train = train.rename(columns={'lat': '6-lat', 'long': '6-long'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['0-bank'] = np.uint(test['0-bank'] * 10)\n",
    "train['0-bank'] = np.uint(train['0-bank'] * 10)\n",
    "columns_for_drop = ['address', 'address_rus']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Запоминаем индексы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['is_train'] = True\n",
    "test['is_train'] = False\n",
    "test['index'] = test.index + 1\n",
    "train['index'] = train.index + 1\n",
    "train['train_index'] = train.index + 1\n",
    "test['test_index'] = test.index + 1\n",
    "train['test_index'] = 0\n",
    "test['train_index'] = 0\n",
    "train['index'] = train.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Создаём общие данные tran-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all = train.append(test, sort=False)\n",
    "\n",
    "data_all['address_fail'] = 0\n",
    "data_all.at[data_all['address_rus'].isnull(), 'address_fail'] = 1\n",
    "data_all['base_target'] = data_all['target']\n",
    "data_all.at[(data_all['address_fail'] == 1) & (data_all['is_train']), 'target'] = 0\n",
    "\n",
    "data_all['address_rus'] = data_all['address_rus'].fillna('')\n",
    "\n",
    "counts = data_all.groupby('address_rus')['id'].count().reset_index().rename(columns={'id':'the_same_address_terminals'})\n",
    "data_all = pd.merge(data_all, counts, how='left', on='address_rus')\n",
    "data_all.at[data_all['address_fail'] == 1, 'the_same_address_terminals'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fillna for address_rus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all.at[data_all['address_fail'], 'address_rus'] = data_all['address_rus'].fillna(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fillna for lat, long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all.at[:, ['6-lat', '6-long']] = data_all[['6-lat', '6-long']].fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fill missed coords\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "missed_ids = [item[3] for item in missed_coords]\n",
    "missed_cities = [item[2] for item in missed_coords]\n",
    "missed_levels = [1 if item[1] == 'Level.HOUSE' else 0 for item in missed_coords]\n",
    "missed_lats = [item[0]['lat'] if item[0] is not None else None for item in missed_coords]\n",
    "missed_longs = [item[0]['long'] if item[0] is not None else None for item in missed_coords]\n",
    "\n",
    "set_missed_ids = set(missed_ids)\n",
    "missed_inds = data_all['id'].apply(lambda el: el in set_missed_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all.at[missed_inds, '6-lat'] = missed_lats\n",
    "data_all.at[missed_inds, '6-long'] = missed_longs\n",
    "data_all['6-levels'] = 1\n",
    "data_all.at[missed_inds, '6-levels'] = missed_levels\n",
    "data_all.at[missed_inds, 'city'] = ['' if city is None else city.lower() for city in missed_cities]\n",
    "data_all['address_fail'] = 0\n",
    "data_all['address_fail'] = data_all['6-lat'].isnull()\n",
    "data_all.at[:, ['6-lat', '6-long']] = data_all[['6-lat', '6-long']].fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Детекция городов в address_rus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_parse_address(row):\n",
    "    if len(row) == 0:\n",
    "        return 'NULL'\n",
    "    d = dict([(item[1], item[0]) for item in parse_address(row)])\n",
    "    city = ''\n",
    "    if 'city' in d:\n",
    "        city = d['city']\n",
    "    return city\n",
    "\n",
    "data_all['city-temp'] = data_all['address_rus'].apply(custom_parse_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_all.loc[missed_inds, 'city-temp'] = data_all[missed_inds]['city']\n",
    "data_all['city'] = data_all['city-temp']\n",
    "data_all['city'] = list(map(lambda x: x.replace('-', ' ').lower().replace('город ', '').replace(' город', '').replace('ё', 'e'), data_all['city']))\n",
    "del data_all['city-temp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Не задетектились\n",
    "\n",
    "d = {\n",
    "    60: 'ст дружба-1',\n",
    "    299: 'буденновский лес',\n",
    "    300: 'буденновский лес',\n",
    "    4443: 'екатеринбург',\n",
    "    4444: 'екатеринбург',\n",
    "    4445: 'екатеринбург',\n",
    "    4446: 'екатеринбург',\n",
    "    8374: 'усть-илимск',\n",
    "    8386: 'усть-илимск',\n",
    "    4609: 'усть-илимск'\n",
    "}\n",
    "for k in d:\n",
    "    data_all.at[k, 'city'] = d[k]\n",
    "data_all.at[data_all['city'] == '', 'city'] = data_all[data_all['city'] == ''].index.astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Парсинг Яндекса"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "  0%|          | 0/8765 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8554/8765 [00:00<00:00, 22869.54it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8554/8765 [00:13<00:00, 22869.54it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8580/8765 [00:14<00:29,  6.27it/s]   \u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8581/8765 [00:14<00:48,  3.78it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8582/8765 [00:15<01:32,  1.99it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8583/8765 [00:16<01:52,  1.62it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8584/8765 [00:17<01:52,  1.60it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8585/8765 [00:18<02:23,  1.25it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8586/8765 [00:19<02:10,  1.37it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8587/8765 [00:19<01:53,  1.57it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8588/8765 [00:19<01:37,  1.81it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8589/8765 [00:20<01:43,  1.69it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8590/8765 [00:21<01:42,  1.71it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8591/8765 [00:21<01:40,  1.73it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8592/8765 [00:22<01:32,  1.88it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8593/8765 [00:22<01:33,  1.84it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8594/8765 [00:23<01:25,  1.99it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8595/8765 [00:23<01:32,  1.84it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8596/8765 [00:24<01:46,  1.58it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8597/8765 [00:25<01:40,  1.66it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8598/8765 [00:25<01:51,  1.50it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8599/8765 [00:27<02:47,  1.01s/it]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8600/8765 [00:28<02:33,  1.08it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8601/8765 [00:28<02:06,  1.30it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8602/8765 [00:29<01:53,  1.44it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8603/8765 [00:30<02:36,  1.04it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8604/8765 [00:31<02:11,  1.22it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8605/8765 [00:32<02:11,  1.22it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8606/8765 [00:32<01:47,  1.49it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8607/8765 [00:33<01:39,  1.59it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8608/8765 [00:33<01:27,  1.79it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8609/8765 [00:34<01:26,  1.80it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8610/8765 [00:34<01:32,  1.67it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8611/8765 [00:35<01:39,  1.55it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8612/8765 [00:36<01:35,  1.60it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8613/8765 [00:36<01:32,  1.64it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8614/8765 [00:37<01:38,  1.53it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8615/8765 [00:37<01:32,  1.63it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8616/8765 [00:38<01:35,  1.56it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8617/8765 [00:39<01:32,  1.61it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8618/8765 [00:39<01:26,  1.71it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8619/8765 [00:40<01:29,  1.63it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8620/8765 [00:40<01:26,  1.67it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8621/8765 [00:41<01:26,  1.66it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8622/8765 [00:42<01:38,  1.45it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8623/8765 [00:42<01:31,  1.56it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8624/8765 [00:43<01:32,  1.53it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8625/8765 [00:44<01:34,  1.48it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8626/8765 [00:45<01:39,  1.40it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8627/8765 [00:45<01:33,  1.48it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8628/8765 [00:46<01:25,  1.61it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8629/8765 [00:46<01:15,  1.79it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8630/8765 [00:47<01:25,  1.58it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8631/8765 [00:48<01:30,  1.48it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8632/8765 [00:48<01:22,  1.62it/s]\u001b[A\u001b[A\n",
      "\n",
      " 98%|█████████▊| 8633/8765 [00:49<01:15,  1.75it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8634/8765 [00:49<01:13,  1.77it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8635/8765 [00:50<01:17,  1.68it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8636/8765 [00:50<01:11,  1.81it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8637/8765 [00:51<01:10,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8638/8765 [00:51<01:09,  1.83it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8639/8765 [00:52<01:00,  2.09it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8640/8765 [00:52<00:56,  2.21it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8641/8765 [00:53<00:58,  2.14it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8642/8765 [00:53<01:03,  1.92it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8643/8765 [00:54<00:56,  2.17it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8644/8765 [00:55<01:16,  1.59it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8645/8765 [00:55<01:13,  1.63it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8646/8765 [00:56<01:07,  1.76it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8647/8765 [00:56<01:06,  1.76it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8648/8765 [00:57<01:01,  1.90it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8649/8765 [00:58<01:23,  1.40it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8650/8765 [00:59<01:20,  1.44it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8651/8765 [00:59<01:10,  1.62it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8652/8765 [00:59<01:06,  1.70it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8653/8765 [01:00<01:05,  1.71it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8654/8765 [01:00<01:00,  1.83it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▊| 8655/8765 [01:01<01:11,  1.54it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8656/8765 [01:02<01:06,  1.63it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8657/8765 [01:03<01:09,  1.56it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8658/8765 [01:03<00:59,  1.80it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8659/8765 [01:04<00:58,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8660/8765 [01:04<00:55,  1.90it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8661/8765 [01:04<00:51,  2.03it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8662/8765 [01:05<00:48,  2.13it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8663/8765 [01:05<00:48,  2.11it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8664/8765 [01:06<00:47,  2.13it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8665/8765 [01:07<01:04,  1.55it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8666/8765 [01:08<01:06,  1.50it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8667/8765 [01:08<01:06,  1.46it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8668/8765 [01:09<00:59,  1.63it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8669/8765 [01:09<00:54,  1.78it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8670/8765 [01:10<00:51,  1.84it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8671/8765 [01:11<01:00,  1.55it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8672/8765 [01:11<00:57,  1.61it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8673/8765 [01:12<00:54,  1.70it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8674/8765 [01:12<00:46,  1.97it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8675/8765 [01:13<01:00,  1.49it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8676/8765 [01:14<00:57,  1.54it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8677/8765 [01:14<00:59,  1.47it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8678/8765 [01:15<00:53,  1.64it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8679/8765 [01:16<01:05,  1.32it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8680/8765 [01:16<00:58,  1.45it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8681/8765 [01:17<00:53,  1.56it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8682/8765 [01:17<00:50,  1.65it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8683/8765 [01:18<00:48,  1.69it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8684/8765 [01:19<00:47,  1.71it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8685/8765 [01:19<00:48,  1.64it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8686/8765 [01:20<00:45,  1.73it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8687/8765 [01:20<00:44,  1.77it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8688/8765 [01:21<00:43,  1.76it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8689/8765 [01:21<00:41,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8690/8765 [01:22<00:41,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8691/8765 [01:23<00:42,  1.75it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8692/8765 [01:23<00:41,  1.76it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8693/8765 [01:23<00:36,  1.97it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8694/8765 [01:24<00:34,  2.04it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8695/8765 [01:25<00:36,  1.94it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8696/8765 [01:25<00:36,  1.88it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8697/8765 [01:26<00:39,  1.73it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8698/8765 [01:26<00:36,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8699/8765 [01:27<00:35,  1.84it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8700/8765 [01:27<00:35,  1.83it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8701/8765 [01:28<00:34,  1.85it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8702/8765 [01:28<00:35,  1.76it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8703/8765 [01:29<00:40,  1.52it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8704/8765 [01:30<00:39,  1.53it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8705/8765 [01:30<00:34,  1.72it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8706/8765 [01:31<00:30,  1.93it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8707/8765 [01:31<00:27,  2.09it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8708/8765 [01:32<00:28,  2.02it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8709/8765 [01:32<00:25,  2.16it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8710/8765 [01:32<00:24,  2.25it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8711/8765 [01:33<00:25,  2.13it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8712/8765 [01:34<00:25,  2.11it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8713/8765 [01:34<00:26,  1.98it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8714/8765 [01:35<00:39,  1.30it/s]\u001b[A\u001b[A\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 8715/8765 [01:36<00:37,  1.32it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8716/8765 [01:37<00:36,  1.33it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8717/8765 [01:38<00:37,  1.29it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8718/8765 [01:39<00:36,  1.28it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8719/8765 [01:39<00:30,  1.52it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8720/8765 [01:40<00:34,  1.30it/s]\u001b[A\u001b[A\n",
      "\n",
      " 99%|█████████▉| 8721/8765 [01:40<00:29,  1.48it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8722/8765 [01:41<00:27,  1.59it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8723/8765 [01:42<00:28,  1.46it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8724/8765 [01:43<00:29,  1.41it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8725/8765 [01:43<00:26,  1.53it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8726/8765 [01:44<00:23,  1.64it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8727/8765 [01:44<00:23,  1.61it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8728/8765 [01:45<00:23,  1.56it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8729/8765 [01:45<00:21,  1.65it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8730/8765 [01:46<00:20,  1.70it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8731/8765 [01:46<00:18,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8732/8765 [01:47<00:18,  1.75it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8733/8765 [01:47<00:16,  1.95it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8734/8765 [01:48<00:16,  1.90it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8735/8765 [01:49<00:17,  1.72it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8736/8765 [01:49<00:15,  1.90it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8737/8765 [01:50<00:14,  1.96it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8738/8765 [01:50<00:13,  2.04it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8739/8765 [01:50<00:12,  2.09it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8740/8765 [01:51<00:11,  2.09it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8741/8765 [01:51<00:11,  2.09it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8742/8765 [01:52<00:11,  2.04it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8743/8765 [01:52<00:11,  1.97it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8744/8765 [01:53<00:11,  1.85it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8745/8765 [01:54<00:10,  1.93it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8746/8765 [01:54<00:09,  1.96it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8747/8765 [01:55<00:10,  1.73it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8748/8765 [01:55<00:08,  2.01it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8749/8765 [01:56<00:08,  1.83it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8750/8765 [01:57<00:09,  1.58it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8751/8765 [01:57<00:09,  1.52it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8752/8765 [01:58<00:09,  1.33it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8753/8765 [01:59<00:07,  1.56it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8754/8765 [01:59<00:07,  1.46it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8755/8765 [02:00<00:06,  1.58it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8756/8765 [02:01<00:06,  1.40it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8757/8765 [02:02<00:06,  1.15it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8758/8765 [02:03<00:05,  1.28it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8759/8765 [02:03<00:04,  1.41it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8760/8765 [02:04<00:03,  1.42it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8761/8765 [02:04<00:02,  1.67it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8762/8765 [02:05<00:01,  1.79it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8763/8765 [02:05<00:01,  1.68it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|█████████▉| 8764/8765 [02:06<00:00,  1.82it/s]\u001b[A\u001b[A\n",
      "\n",
      "100%|██████████| 8765/8765 [02:07<00:00,  1.45it/s]\u001b[A\u001b[A\n",
      "\n",
      "\u001b[A\u001b[A"
     ]
    }
   ],
   "source": [
    "for i, addr in tqdm(enumerate(data_all['address_rus'].values), total=data_all.shape[0]):\n",
    "    if i < 8553:\n",
    "        continue\n",
    "    resp = get_items('uohiuhiguyfytdtrcrytfugyug87t87t86r64ded6' if addr == '' else addr)\n",
    "    with open('yandex/data/' + str(i) + '.json', 'w') as fd:\n",
    "        json.dump(resp, fd)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}