1.0 - Thesaurus - Split WikiFile.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IMPORTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cfg\n",
    "\n",
    "import gzip\n",
    "import os\n",
    "from progress_bar import ProgressBar\n",
    "from progress_bar.utils import iter_progress\n",
    "from progress_bar.labeling import get_custom_labeling_fun"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CHECK IF ALL THE RAW FILES EXIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2\n",
    "assert os.path.isfile(cfg.raw_dir + \"enwiki-latest-pages-articles.xml.gz\")\n",
    "# https://www.mediawiki.org/wiki/Wikibase/DataModel/JSON#Claims_and_Statements\n",
    "assert os.path.isfile(cfg.raw_dir + \"wikidata-all.json.gz\")\n",
    "# http://www.openoffice.org/lingucomponent/thesaurus.html\n",
    "assert os.path.isfile(cfg.raw_dir + \"thesaurus_en_openoffice_v1.txt.gz\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CREATE THE OUTPUT FILES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wiki_parts_path = cfg.processed_dir + \"enwiki-latest-pages-articles_parts/\"\n",
    "if not os.path.exists(wiki_parts_path):\n",
    "    os.mkdir(wiki_parts_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "outfiles = [\n",
    "    gzip.open(\"{}part_{}_{}.xml.gz\".format(wiki_parts_path, i+1, cfg.wiki_preprocessing_split_into), \"w\")\n",
    "    for i in range(cfg.wiki_preprocessing_split_into)\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FILL THE OUTPUT FILES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2\n",
    "assert os.path.isfile(cfg.raw_dir + \"enwiki-latest-pages-articles.xml.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index = 0\n",
    "\n",
    "buffer_rows = []\n",
    "buffer_chars = 0\n",
    "buffer_num_pages = 0\n",
    "buffer_max_chars = 8 * 1024 * 1024\n",
    "\n",
    "pb = ProgressBar(labeling_fun=get_custom_labeling_fun(prefix=\"Pages\", show_remaining_time=False)).display()\n",
    "with gzip.open(cfg.raw_dir + \"enwiki-latest-pages-articles.xml.gz\", \"r\") as infile:\n",
    "    for line in iter_progress(infile, labeling_fun=get_custom_labeling_fun(prefix=\"Rows\", show_remaining_time=False)):\n",
    "        if line == \"  <page>\\n\":\n",
    "            buffer_num_pages += 1\n",
    "\n",
    "            if buffer_chars >= buffer_max_chars:\n",
    "                outfiles[index].write(\"\".join(buffer_rows))\n",
    "                index = (index + 1) % cfg.wiki_preprocessing_split_into\n",
    "                pb.increase_many(buffer_num_pages)\n",
    "\n",
    "                buffer_rows = []\n",
    "                buffer_chars = 0\n",
    "                buffer_num_pages = 0\n",
    "\n",
    "        buffer_rows.append(line)\n",
    "        buffer_chars += len(line)                \n",
    "\n",
    "    # write the reamining part\n",
    "    if buffer_chars > 0:\n",
    "        outfiles[index].write(\"\".join(buffer_rows))\n",
    "        pb.increase_many(buffer_num_pages)\n",
    "    pb.stop(True)\n",
    "\n",
    "# Summary: 17.152.607 Pages - 938.636.136 Rows (8115s - 2h 15m 15s)\n",
    "# it lasts 2h 15m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CLOSE THE OUTPUT FILES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for outfile in outfiles:\n",
    "    outfile.close()\n",
    "outfiles = []"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}