Get_DTP_FederalHighways.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76051662",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests as r\n",
    "from json import loads,dumps\n",
    "import pandas as pd\n",
    "from numpy import random, array_split\n",
    "from multiprocessing.pool import ThreadPool\n",
    "import time\n",
    "import re\n",
    "import os\n",
    "\n",
    "months=['MONTHS:12.2015','MONTHS:12.2016','MONTHS:12.2017','MONTHS:12.2018','MONTHS:12.2019','MONTHS:12.2020','MONTHS:12.2021','MONTHS:12.2022','MONTHS:12.2023','MONTHS:1.2024']\n",
    "\n",
    "\n",
    "\n",
    "desktop_agents = ['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14',\n",
    "                 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36',\n",
    "                 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0']\n",
    "inc=50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "17c354ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Get_OKTMOS():\n",
    "    regions_dict = {\"maptype\":1,\"region\":\"877\",\"date\":\"[\\\"11.2020\\\"]\",\"pok\":\"1\"}\n",
    "    k=1\n",
    "    while k==1:\n",
    "        try:\n",
    "            raw_data = r.post(\"http://stat.gibdd.ru/map/getMainMapData\", json=regions_dict)\n",
    "            k=0\n",
    "        except:\n",
    "            time.sleep(10)\n",
    "    raw_reg_js=loads(raw_data.content)\n",
    "    raw_reg_js=loads(loads(raw_reg_js[\"metabase\"])[0][\"maps\"])\n",
    "    oktmos=[(i[\"id\"],i[\"name\"]) for i in raw_reg_js]\n",
    "    region=[]\n",
    "    region_oktmo=[]\n",
    "    district=[]\n",
    "    district_oktmo=[]\n",
    "    for oktmo in oktmos:\n",
    "        subregions_dict = {\"maptype\":1,\"region\":\"877\",\"date\":\"[\\\"11.2020\\\"]\",\"pok\":\"1\",\"region\":oktmo[0]}\n",
    "        raw_subdata=r.post(\"http://stat.gibdd.ru/map/getMainMapData\", json=subregions_dict)\n",
    "        raw_subreg_js=loads(loads(loads(raw_subdata.content)[\"metabase\"])[0][\"maps\"])\n",
    "        for js in raw_subreg_js:\n",
    "            region.append(oktmo[1])\n",
    "            region_oktmo.append(oktmo[0])\n",
    "            district.append(js[\"name\"])\n",
    "            district_oktmo.append(js[\"id\"])\n",
    "\n",
    "    df=pd.DataFrame([region,district,region_oktmo,district_oktmo]).T\n",
    "    df.columns=[\"Region\",\"District\",\"Region_OKTMO\",\"District_OKTMO\"]\n",
    "    df.to_excel(\"OKTMOS.xlsx\",index=False)\n",
    "Get_OKTMOS()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c25058df",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Get_Road_Info():\n",
    "    raw_data=r.post(\"http://stat.gibdd.ru/road/getMainMapData\")\n",
    "    data=loads(loads(raw_data.content)[\"metabase\"])[\"features\"]\n",
    "    ids=[]\n",
    "    for i in data:\n",
    "        ids.append((i[\"properties\"][\"id\"],i[\"properties\"][\"name\"]))\n",
    "    ids=ids[:85]\n",
    "    mega_roads=[]\n",
    "    for id_ in ids:\n",
    "        roads=[]\n",
    "        raw_data1=r.post(\"http://stat.gibdd.ru/road/getRegionDorCombo\",json={\"reg\":id_[0]})\n",
    "        try:\n",
    "            raw_data1=loads(raw_data1.content)\n",
    "            for rd in raw_data1:\n",
    "                if (\"ИСКЛЮЧЕНА\" in rd[\"text\"]) or (\"исключена\" in rd[\"text\"]):\n",
    "                    pass\n",
    "                else:\n",
    "                    roads.append((rd[\"value\"],rd[\"text\"]))\n",
    "        except:\n",
    "            roads.append((\"0\",\"0\"))\n",
    "        mega_roads.append(roads)\n",
    "    mega_tuples=[]\n",
    "    for ind,id_ in enumerate(ids):\n",
    "        if (mega_roads[ind]==[(\"0\",\"0\")]) or (mega_roads[ind]==[]):\n",
    "            pass\n",
    "        else:\n",
    "            mega_tuples.append([id_,mega_roads[ind]])\n",
    "    reg_numbers=[]\n",
    "    reg_names=[]\n",
    "    road_numbers=[]\n",
    "    road_names=[]\n",
    "    for tup in mega_tuples:\n",
    "        reg_info,roads=tup\n",
    "        for road in roads:\n",
    "            reg_numbers.append(reg_info[0])\n",
    "            reg_names.append(reg_info[1])\n",
    "            road_value,road_text=road\n",
    "            road_numbers.append(road_value)\n",
    "            road_names.append(road_text)\n",
    "    df=pd.DataFrame([reg_numbers,reg_names,road_numbers,road_names]).T\n",
    "    df.columns=[\"Region_Number\",\"Region\",\"Road_Number\",\"Road\"]\n",
    "    df.to_excel(\"Roads.xlsx\",index=False)\n",
    "    return df\n",
    "df1=Get_Road_Info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b6e16323",
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_post(js,f0,t1,t2):\n",
    "    u=0\n",
    "    while u==0:\n",
    "        try:\n",
    "            post_content=f0.post(\"http://stat.gibdd.ru/road/getDorKMData\",json=js,headers={\"User-Agent\":random.choice(desktop_agents)},verify=False)\n",
    "            u=1\n",
    "        except:\n",
    "            time.sleep(10)\n",
    "            print(\"Таймаут {},{}\".format(t1,t2))\n",
    "    return post_content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f3abe6e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_km_months(rows_part):\n",
    "    num,rows_part=rows_part\n",
    "    km_and_months=[]\n",
    "    with r.Session() as f:\n",
    "        for ind,part in enumerate(rows_part):\n",
    "            indexes=list(df1[df1.Region==part].index)\n",
    "            for index in indexes:\n",
    "                x=0\n",
    "                tup=tuple(df1.iloc[index,:])\n",
    "                for month in months:\n",
    "                    dicts={\"dor\":str(tup[2]),\"reg\":str(tup[0]),\"date\":month}\n",
    "                    raw_data=do_post(dicts,f,tup[1],tup[3])\n",
    "                    raw_data=raw_data.json()[\"data\"]\n",
    "                    x+=len(raw_data)\n",
    "                    for d in raw_data:\n",
    "                        km_and_months.append((tup[3],tup[1],tup[2],d[\"name\"],month))\n",
    "\n",
    "    return km_and_months"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e36f6dc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_km_frame():\n",
    "    regs=list(df1.Region.unique())\n",
    "    rows_divided=list(array_split(regs,4))\n",
    "    nums=[1,2,3,4]\n",
    "    rows_divided=list(zip(nums,rows_divided))\n",
    "    pool=ThreadPool(4)\n",
    "    l=pool.map(get_km_months,rows_divided)\n",
    "    res=[]\n",
    "    for l_l in l:\n",
    "        for l_l_l in l_l:\n",
    "            res.append(l_l_l)\n",
    "    names=[i[0] for i in res]\n",
    "    regs=[i[1] for i in res]\n",
    "    road_nums=[i[2] for i in res]\n",
    "    km_nums=[i[3] for i in res]\n",
    "    months=[i[4] for i in res]\n",
    "    df=pd.DataFrame([names,regs,road_nums,km_nums,months]).T\n",
    "    df.columns=[\"Name\",\"Region\",\"Road_Num\",\"KM\",\"Month\"]\n",
    "    df.to_excel(\"Stats_by_region_roads.xlsx\",index=False)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "344b8dfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_post_card(js,f0,t1,t2):\n",
    "    u=0\n",
    "    while u==0:\n",
    "        try:\n",
    "            post_content=f0.post(\"http://stat.gibdd.ru/road/getDorKMKardList\",json=js,headers={\"User-Agent\":random.choice(desktop_agents)},verify=False)\n",
    "            u=1\n",
    "        except:\n",
    "            time.sleep(10)\n",
    "            print(\"Таймаут {},{}\".format(t1,t2))\n",
    "    return post_content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "803cf98b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dtp_cards(rows_divided):\n",
    "    num,rows_part,df2=rows_divided\n",
    "    kard_data=[]\n",
    "    with r.Session() as fk:\n",
    "        for ind,part in enumerate(rows_part):\n",
    "            indexes=list(df2[df2.Region==part].index)\n",
    "            x=0\n",
    "            for index in indexes:\n",
    "                tup=tuple(df2.iloc[index,:])\n",
    "                st=1\n",
    "                j=True\n",
    "                while j:\n",
    "                    dicts={\"data\":{\"date\":[tup[-1]],\"order\":{\"type\":\"1\",\"fieldName\":\"dat\"},\"dor\":str(tup[2]),\"km\":str(tup[3]),\"st\":str(st),\"en\":str(st+inc)}}\n",
    "                    js={}\n",
    "                    js[\"data\"]=dumps(dicts[\"data\"],separators=(',', ':')).encode(\"utf8\").decode('unicode-escape')\n",
    "                    raw_data= do_post_card(js,fk,tup[1],tup[0])\n",
    "                    try:\n",
    "                        data=loads(loads(raw_data.content)[\"data\"])[\"tab\"]\n",
    "                        st=st+inc+1\n",
    "                        x+=len(data)\n",
    "                        for i in data:\n",
    "                            road=i[\"infoDtp\"][\"dor\"]\n",
    "                            date=i[\"date\"]\n",
    "                            district=i[\"District\"]\n",
    "                            reg=part\n",
    "                            curr_km=tup[3]\n",
    "                            type_acc=i[\"DTP_V\"]\n",
    "                            timing=i['Time']\n",
    "                            identif=i['KartId']\n",
    "                            deaths=i['POG']\n",
    "                            wounded=i['RAN']\n",
    "                            partic=i['K_UCH']\n",
    "                            nofcars=i['K_TS']\n",
    "                            street_type=i['infoDtp']['k_ul']\n",
    "                            factor=\" \"\n",
    "                            for f in i['infoDtp']['factor']:\n",
    "                                factor+=f+\" \"\n",
    "                            weather=\" \"\n",
    "                            for f in i['infoDtp']['s_pog']:\n",
    "                                weather+=f+\" \"\n",
    "                            spch=i['infoDtp']['s_pch']\n",
    "                            brightn=i['infoDtp']['osv']\n",
    "                            change_motion=i['infoDtp']['change_org_motion']\n",
    "                            nedostatk=\" \"\n",
    "                            for f in i['infoDtp']['ndu']:\n",
    "                                nedostatk+=f+\" \"\n",
    "                            UDS_NA_DTP=\" \"\n",
    "                            for f in i['infoDtp']['sdor']:\n",
    "                                UDS_NA_DTP+=f+\" \"\n",
    "                        \n",
    "                            UDS_NEAR_DTP=\" \"\n",
    "                            for f in i['infoDtp']['OBJ_DTP']:\n",
    "                                UDS_NEAR_DTP+=f+\" \"\n",
    "                            kard_data.append((road,date,district,reg,curr_km,type_acc,timing,identif,deaths,wounded,partic,nofcars,street_type,factor,\n",
    "                                              weather,spch,brightn,change_motion,nedostatk,UDS_NA_DTP,UDS_NEAR_DTP))\n",
    "                    except:\n",
    "                        j=False\n",
    "            print(part,x)\n",
    "            time.sleep(20)\n",
    "    return kard_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "709e5c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Federal_Highways_DTP_Parser():\n",
    "    df2=make_km_frame()\n",
    "    regs=list(df2.Region.unique())\n",
    "    rows_divided=list(array_split(regs,4))\n",
    "    nums=[1,2,3,4]\n",
    "    rd=list(zip(nums,rows_divided))\n",
    "    rd_n=[]\n",
    "    for r1 in rd:\n",
    "        rd_n.append((r1[0],r1[1],df2))\n",
    "    pool=ThreadPool(4)\n",
    "    l=pool.map(get_dtp_cards,rd_n)\n",
    "    tuples=[]\n",
    "    for l_l in l:\n",
    "        for l_l_l in l_l:\n",
    "            tuples.append(l_l_l)\n",
    "    print(\"Парсинг закончен.Работаю с данными\")\n",
    "    roads=[i[0] for i in tuples]\n",
    "    dates=[i[1] for i in tuples]\n",
    "    districts=[i[2] for i in tuples]\n",
    "    regs=[i[3] for i in tuples]\n",
    "    kms=[i[4] for i in tuples]\n",
    "    types=[i[5] for i in tuples]\n",
    "    timings=[i[6] for i in tuples]\n",
    "    ids=[i[7] for i in tuples]\n",
    "    deaths=[i[8] for i in tuples]\n",
    "    woundeds=[i[9] for i in tuples]\n",
    "    participants=[i[10] for i in tuples]\n",
    "    nofcars=[i[11] for i in tuples]\n",
    "    street_types=[i[12] for i in tuples]\n",
    "    factors=[i[13] for i in tuples]\n",
    "    weathers=[i[14] for i in tuples]\n",
    "    spchs=[i[15] for i in tuples]\n",
    "    brightns=[i[16] for i in tuples]\n",
    "    change_motion=[i[17] for i in tuples]\n",
    "    nedostatk=[i[18] for i in tuples]\n",
    "    uds_na_dtp=[i[19] for i in tuples]\n",
    "    uds_near_dtp=[i[20] for i in tuples]\n",
    "    df_f=pd.DataFrame([ids,roads,dates,timings,districts,regs,kms,street_types,types,nofcars,participants,deaths,woundeds,factors,\n",
    "                      weathers,spchs,brightns,change_motion,nedostatk,uds_na_dtp,uds_near_dtp]).T\n",
    "    df_f.columns=['Accident ID',\"Road\",\"Date\",\"Time\",\"District\",\"Region\",\"KM\",\"Street type\",\"Accident type\",\"#,cars\",\"#,participants\",\"#,deaths\",\"#,wounded\",\"Accident factor\"\n",
    "                 ,\"Weather\",\"Road condition\",\"Lighting\",\"Changes in dr.mode\",\"Disadv.of road network\",\"UDS on place\",\"UDS near place\"]\n",
    "    df_f[\"Road_Abbr\"]=df_f[\"Road\"].map(lambda x:re.findall(r\"[АМР]-\\d{1,4}\\s?\",x))\n",
    "    df_f[\"Road_Abbr\"]=[i[0] if len(i)>0 else \"\" for i in df_f.Road_Abbr]\n",
    "    df_f.loc[df_f.Road==\"Крым Москва - Тула - Орел - Курск - Белгород - граница с Украиной, подъезд к заповеднику Прохоровское поле\",\"Road_Abbr\"]=\"М-2\"\n",
    "    df_f.loc[df_f.Road==\"Самара – Пугачёв – Энгельс - Волгоград\",\"Road_Abbr\"]=\"Р-226\"\n",
    "    df_f.loc[df_f.Road==\"Чекшино - Тотьма - Котлас - Куратово\",\"Road_Abbr\"]=\"А-123\"\n",
    "    df_f.loc[df_f.Road==\"Обход г. Нижнего Новгорода\",\"Road_Abbr\"]=\"М-7\"\n",
    "    df_f.loc[df_f.Road==\"Кочубей-Зеленокумск через Нефтекумск\",\"Road_Abbr\"]=\"А-167\"\n",
    "    df_f.loc[df_f.Road==\"Объезд г. Бийска\",\"Road_Abbr\"]=\"Р-256\"\n",
    "    df_f.loc[df_f.Road==\"Барнаул - Павловск - граница с Республикой Казахстан\",\"Road_Abbr\"]=\"А-321\"\n",
    "    df_f.loc[df_f.Road==\"1Р 418  Иркутск - Усть-Ордынский\",\"Road_Abbr\"]=\"Р-418\"\n",
    "    df_f.loc[df_f.Road==\"Косолаповы – Новоурожайная – Наймушины (объезд г. Котельнича в составе а/д «Вятка»)\",\"Road_Abbr\"]=\"Р-176\"\n",
    "    df_f.loc[df_f.Road.isin([\"Внешняя сторона\",\"Внутренняя сторона\"]),\"Road_Abbr\"]=\"МКАД\"\n",
    "    df_f.loc[df_f.Road==\"«Таврида» Керчь – Симферополь - Севастополь\",\"Road_Abbr\"]=\"А-291\"\n",
    "    df_f.loc[df_f.Road==\"Южно-Сахалинск - Оха\",\"Road_Abbr\"]=\"А-393\"\n",
    "    df_f.Road_Abbr=df_f.Road_Abbr.str.rstrip().str.lstrip()\n",
    "    df_f[\"KM\"]=df_f[\"KM\"].astype(\"category\")\n",
    "    df_f[\"Accident type\"]=df_f[\"Accident type\"].astype(\"category\")\n",
    "    df_f[\"Date\"]=pd.to_datetime(df_f[\"Date\"],format=\"%d.%m.%Y\")\n",
    "    print(\"Сохраняю\")\n",
    "    df_f.to_excel(\"Federal_Highways_Accidents.xlsx\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6f6f5ff1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Республика Марий Эл 1021\n",
      "Курская область 1576\n",
      "Белгородская область 562\n",
      "Пензенская область 2994\n",
      "Республика Дагестан 4835\n",
      "Орловская область 1722\n",
      "Калининградская область 590\n",
      "Республика Татарстан (Татарстан) 4121\n",
      "Астраханская область 1128\n",
      "Саратовская область 3827\n",
      "Республика Ингушетия 444\n",
      "Чувашская Республика - Чувашия 2067\n",
      "Республика Калмыкия 1398\n",
      "Чеченская Республика 491\n",
      "Тульская область 3982\n",
      "Республика Мордовия 1339\n",
      "Республика Карелия 1793\n",
      "Кабардино-Балкарская Республика 2070\n",
      "Липецкая область 2093\n",
      "Удмуртская Республика 1302\n",
      "Республика Хакаси 589\n",
      "Карачаево-Черкесская Республика 1867\n",
      "Тамбовская область 1907\n",
      "Республика Адыгея (Адыгея) 1271\n",
      "Красноярский край 3245\n",
      "Сириус 79\n",
      "Пермский край 1820\n",
      "Республика Крым 271\n",
      "гор. Севастополь 9\n",
      "Амурская область 1479\n",
      "Воронежская область 4125\n",
      " Кемеровска область - Кузбасс 1185\n",
      " Курганская область 1615\n",
      "Волгоградская область 2318\n",
      "Ленинградская область 7646\n",
      "Магаданская область 533\n",
      "Оренбургская область 2475\n",
      "Калужская область 4052\n",
      "Краснодарский край 11083\n",
      "Самарская область 2967\n",
      "Тверская область 2669\n",
      "Ставропольский край 4634\n",
      "Брянская область 1596\n",
      "Свердловская область 2100\n",
      "Томская область 78\n",
      "Республика Алтай 972\n",
      "Ульяновская область 1437\n",
      "Смоленская область 2219\n",
      "Алтайский край 2530\n",
      "Псковская область 1794\n",
      "Челябинская область 2940\n",
      "Московская область 15601\n",
      "Новосибирская область 2039\n",
      "Еврейская автономная область 652\n",
      "Новгородская область 1906\n",
      "Мурманская область 1348\n",
      "Ханты-Мансийский автономный округ - Югра 960\n",
      "Омская область 1474\n",
      "Вологодская область 1977\n",
      "Архангельская область 1488\n",
      "Республика Башкортостан 3395\n",
      "Ярославская область 2427\n",
      "Тюменская область 3413\n",
      "гор. Москва 4815\n",
      "Костромская область 734\n",
      "Ивановская область 213\n",
      "гор. Санкт-Петербург 1891\n",
      "Республика Бурятия 1709\n",
      "Ростовская область 4093\n",
      "Республика Тыва 689\n",
      "Владимирская область 3544\n",
      "Республика Саха (Якутия) 1268\n",
      "Сахалинская область 746\n",
      "Республика Северная Осетия-Алания 1451\n",
      "Иркутская область 3883\n",
      "Рязанская область 3306\n",
      "Приморский край 2580\n",
      "Забайкальский край 2202\n",
      "Нижегородская область 3866\n",
      "Республика Коми 536\n",
      "Хабаровский край 1264\n",
      "Камчатский край 120\n",
      "Кировская область 1629\n",
      "Парсинг закончен.Работаю с данными\n",
      "Сохраняю\n"
     ]
    }
   ],
   "source": [
    "Federal_Highways_DTP_Parser()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}