gen_source.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Extra data: line 1 column 22 - line 34657 column 30 (char 21 - 3027087)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-7336fe855b3c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    447\u001b[0m         \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    448\u001b[0m         \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msub\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"'\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'\"'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 449\u001b[1;33m         \u001b[0mmessages\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    450\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    451\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmessage\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmessages\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python27\\lib\\json\\__init__.pyc\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m    337\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 339\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    340\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\python27\\lib\\json\\decoder.pyc\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m    365\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    366\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 367\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merrmsg\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Extra data\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    368\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Extra data: line 1 column 22 - line 34657 column 30 (char 21 - 3027087)"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import json\n",
    "from pymystem3 import Mystem\n",
    "import nltk\n",
    "import re\n",
    "from codecs import open\n",
    "from json import loads\n",
    "\n",
    "m = Mystem()\n",
    "regexp_pattern = '.!?,;:\\'\\\"\\/\\\\'\n",
    "#stopwords = nltk.corpus.stopwords.words('russian')\n",
    "\n",
    "\n",
    "\n",
    "stop_words = \"\"\"\n",
    "а\n",
    "е\n",
    "и\n",
    "ж\n",
    "м\n",
    "о\n",
    "на\n",
    "не\n",
    "ни\n",
    "об\n",
    "но\n",
    "он\n",
    "мне\n",
    "мои\n",
    "мож\n",
    "она\n",
    "они\n",
    "оно\n",
    "мной\n",
    "много\n",
    "многочисленное\n",
    "многочисленная\n",
    "многочисленные\n",
    "многочисленный\n",
    "мною\n",
    "мой\n",
    "мог\n",
    "могут\n",
    "можно\n",
    "может\n",
    "можхо\n",
    "мор\n",
    "моя\n",
    "моё\n",
    "мочь\n",
    "над\n",
    "нее\n",
    "оба\n",
    "нам\n",
    "нем\n",
    "нами\n",
    "ними\n",
    "мимо\n",
    "немного\n",
    "одной\n",
    "одного\n",
    "менее\n",
    "однажды\n",
    "однако\n",
    "меня\n",
    "нему\n",
    "меньше\n",
    "ней\n",
    "наверху\n",
    "него\n",
    "ниже\n",
    "мало\n",
    "надо\n",
    "один\n",
    "одиннадцать\n",
    "одиннадцатый\n",
    "назад\n",
    "наиболее\n",
    "недавно\n",
    "миллионов\n",
    "недалеко\n",
    "между\n",
    "низко\n",
    "меля\n",
    "нельзя\n",
    "нибудь\n",
    "непрерывно\n",
    "наконец\n",
    "никогда\n",
    "никуда\n",
    "нас\n",
    "наш\n",
    "нет\n",
    "нею\n",
    "неё\n",
    "них\n",
    "мира\n",
    "наша\n",
    "наше\n",
    "наши\n",
    "ничего\n",
    "начала\n",
    "нередко\n",
    "несколько\n",
    "обычно\n",
    "опять\n",
    "около\n",
    "мы\n",
    "ну\n",
    "нх\n",
    "от\n",
    "отовсюду\n",
    "особенно\n",
    "нужно\n",
    "очень\n",
    "отсюда\n",
    "в\n",
    "во\n",
    "вон\n",
    "вниз\n",
    "внизу\n",
    "вокруг\n",
    "вот\n",
    "восемнадцать\n",
    "восемнадцатый\n",
    "восемь\n",
    "восьмой\n",
    "вверх\n",
    "вам\n",
    "вами\n",
    "важное\n",
    "важная\n",
    "важные\n",
    "важный\n",
    "вдали\n",
    "везде\n",
    "ведь\n",
    "вас\n",
    "ваш\n",
    "ваша\n",
    "ваше\n",
    "ваши\n",
    "впрочем\n",
    "весь\n",
    "вдруг\n",
    "вы\n",
    "все\n",
    "второй\n",
    "всем\n",
    "всеми\n",
    "времени\n",
    "время\n",
    "всему\n",
    "всего\n",
    "всегда\n",
    "всех\n",
    "всею\n",
    "всю\n",
    "вся\n",
    "всё\n",
    "всюду\n",
    "г\tгод\n",
    "говорил\n",
    "говорит\n",
    "года\n",
    "году\n",
    "где\n",
    "да\n",
    "ее\n",
    "за\n",
    "из\n",
    "ли\n",
    "же\n",
    "им\n",
    "до\n",
    "по\n",
    "ими\n",
    "под\n",
    "иногда\n",
    "довольно\n",
    "именно\n",
    "долго\n",
    "позже\n",
    "более\n",
    "должно\n",
    "пожалуйста\n",
    "значит\n",
    "иметь\n",
    "больше\n",
    "пока\n",
    "ему\n",
    "имя\n",
    "пор\n",
    "пора\n",
    "потом\n",
    "потому\n",
    "после\n",
    "почему\n",
    "почти\n",
    "посреди\n",
    "ей\n",
    "два\n",
    "две\n",
    "двенадцать\n",
    "двенадцатый\n",
    "двадцать\n",
    "двадцатый\n",
    "двух\n",
    "его\n",
    "дел\n",
    "или\n",
    "без\n",
    "день\n",
    "занят\n",
    "занята\n",
    "занято\n",
    "заняты\n",
    "действительно\n",
    "давно\n",
    "девятнадцать\n",
    "девятнадцатый\n",
    "девять\n",
    "девятый\n",
    "даже\n",
    "алло\n",
    "жизнь\n",
    "далеко\n",
    "близко\n",
    "здесь\n",
    "дальше\n",
    "для\n",
    "лет\n",
    "зато\n",
    "даром\n",
    "первый\n",
    "перед\n",
    "затем\n",
    "зачем\n",
    "лишь\n",
    "десять\n",
    "десятый\n",
    "ею\n",
    "её\n",
    "их\n",
    "бы\n",
    "еще\n",
    "при\n",
    "был\n",
    "про\n",
    "процентов\n",
    "против\n",
    "просто\n",
    "бывает\n",
    "бывь\n",
    "если\n",
    "люди\n",
    "была\n",
    "были\n",
    "было\n",
    "будем\n",
    "будет\n",
    "будете\n",
    "будешь\n",
    "прекрасно\n",
    "буду\n",
    "будь\n",
    "будто\n",
    "будут\n",
    "ещё\n",
    "пятнадцать\n",
    "пятнадцатый\n",
    "друго\n",
    "другое\n",
    "другой\n",
    "другие\n",
    "другая\n",
    "других\n",
    "есть\n",
    "пять\n",
    "быть\n",
    "лучше\n",
    "пятый\n",
    "к\n",
    "ком\n",
    "конечно\n",
    "кому\n",
    "кого\n",
    "когда\n",
    "которой\n",
    "которого\n",
    "которая\n",
    "которые\n",
    "который\n",
    "которых\n",
    "кем\n",
    "каждое\n",
    "каждая\n",
    "каждые\n",
    "каждый\n",
    "кажется\n",
    "как\n",
    "какой\n",
    "какая\n",
    "кто\n",
    "кроме\n",
    "куда\n",
    "кругом\n",
    "с\tт\n",
    "у\n",
    "я\n",
    "та\n",
    "те\n",
    "уж\n",
    "со\n",
    "то\n",
    "том\n",
    "снова\n",
    "тому\n",
    "совсем\n",
    "того\n",
    "тогда\n",
    "тоже\n",
    "собой\n",
    "тобой\n",
    "собою\n",
    "тобою\n",
    "сначала\n",
    "только\n",
    "уметь\n",
    "тот\n",
    "тою\n",
    "хорошо\n",
    "хотеть\n",
    "хочешь\n",
    "хоть\n",
    "хотя\n",
    "свое\n",
    "свои\n",
    "твой\n",
    "своей\n",
    "своего\n",
    "своих\n",
    "свою\n",
    "твоя\n",
    "твоё\n",
    "раз\n",
    "уже\n",
    "сам\n",
    "там\n",
    "тем\n",
    "чем\n",
    "сама\n",
    "сами\n",
    "теми\n",
    "само\n",
    "рано\n",
    "самом\n",
    "самому\n",
    "самой\n",
    "самого\n",
    "семнадцать\n",
    "семнадцатый\n",
    "самим\n",
    "самими\n",
    "самих\n",
    "саму\n",
    "семь\n",
    "чему\n",
    "раньше\n",
    "сейчас\n",
    "чего\n",
    "сегодня\n",
    "себе\n",
    "тебе\n",
    "сеаой\n",
    "человек\n",
    "разве\n",
    "теперь\n",
    "себя\n",
    "тебя\n",
    "седьмой\n",
    "спасибо\n",
    "слишком\n",
    "так\n",
    "такое\n",
    "такой\n",
    "такие\n",
    "также\n",
    "такая\n",
    "сих\n",
    "тех\n",
    "чаще\n",
    "четвертый\n",
    "через\n",
    "часто\n",
    "шестой\n",
    "шестнадцать\n",
    "шестнадцатый\n",
    "шесть\n",
    "четыре\n",
    "четырнадцать\n",
    "четырнадцатый\n",
    "сколько\n",
    "сказал\n",
    "сказала\n",
    "сказать\n",
    "ту\n",
    "ты\n",
    "три\n",
    "эта\n",
    "эти\n",
    "что\n",
    "это\n",
    "чтоб\n",
    "этом\n",
    "этому\n",
    "этой\n",
    "этого\n",
    "чтобы\n",
    "этот\n",
    "стал\n",
    "туда\n",
    "этим\n",
    "этими\n",
    "рядом\n",
    "тринадцать\n",
    "тринадцатый\n",
    "этих\n",
    "третий\n",
    "тут\n",
    "эту\n",
    "суть\n",
    "чуть\n",
    "тысяч\n",
    "\"\"\"\n",
    "\n",
    "lemmas = []\n",
    "\n",
    "from glob import glob\n",
    "\n",
    "li = glob(\"full-messages/*\")\n",
    "\n",
    "q = []\n",
    "\n",
    "for filename in li:\n",
    "    with open(filename, 'r', encoding='utf-8') as f:\n",
    "        s = f.read()\n",
    "        s = re.sub(\"'\", '\"', s)\n",
    "        messages = json.loads(s)\n",
    "\n",
    "        for message in messages:\n",
    "            subject_regexped = \" \".join([n for n in re.findall(\"[А-ЯЁа-яё]+\", message[\"subject\"].lower()) if not n in stop_words])\n",
    "            snippet_regexped =  \" \".join([n for n in re.findall(\"[А-ЯЁа-яё]+\", message[\"snippet\"].lower()) if not n in stop_words])\n",
    "\n",
    "            subject = ''.join(m.lemmatize(subject_regexped))\n",
    "            snippet = ''.join(m.lemmatize(snippet_regexped))\n",
    "\n",
    "            email = message[\"correspondents\"][\"from\"][0][\"email\"]\n",
    "            message_id = message[\"id\"]\n",
    "\n",
    "            message_stringified = message_id + ' |text ' + subject + ' ' + snippet + ' |author ' + email\n",
    "            lemmas.append(message_stringified)\n",
    "\n",
    "    with open('output/'+filename.split(\"/\")[-1].split(\".\")[0]+\".txt\", 'w', encoding='utf-8') as f:\n",
    "        f.write('\\n'.join(lemmas))\n",
    "\n",
    "    q += lemmas\n",
    "\n",
    "    print(filename)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}