practice_psk_040421.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from packages.metagenomics import sampling2, encoding2, naive_bayes\n",
    "from Bio import SeqIO\n",
    "import numpy as np\n",
    "from Bio.Seq import Seq\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test concat_cols(): PASS\n"
     ]
    }
   ],
   "source": [
    "def concat_cols(row):\n",
    "    l = row.tolist()\n",
    "    fragment = l[:-1]\n",
    "    taxid = l[-1]\n",
    "    sequence = ''.join(fragment)\n",
    "    return [sequence, taxid]\n",
    "\n",
    "print('test concat_cols(): PASS')\n",
    "test_a = np.array(['1','2','3','4','5'])\n",
    "test_row = concat_cols(test_a)\n",
    "assert test_row[0] == '1234'\n",
    "assert test_row[1] == '5'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 116 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "seq_file ='data/train_small-db_toy-2000.fasta'\n",
    "taxid_file = 'data/train_small-db_toy-2000.taxid'\n",
    "output_dir = 'data/sampling/sampling-toy-2000-2'\n",
    "sample_len = 20\n",
    "coverage = 1\n",
    "seed = 42\n",
    "\n",
    "sampling2.generate_fragment_data(seq_file, taxid_file, output_dir, sample_len, coverage, seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(409, 21)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_dir ='data/sampling/sampling-toy-2000-2'\n",
    "pattern = 'fragments*.npy'\n",
    "\n",
    "fragments_toy = sampling2.read_fragments(input_dir, pattern)\n",
    "fragments_toy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['g', 'c', 'a', 'g', 't', 't', 'a', 'a', 'a', 'c', 'c', 'c', 'g', 'a', 'c', 'g', 'c', 'a', 't', 'a', '1280']\n",
      "['t', 't', 'g', 'a', 't', 't', 'g', 'a', 't', 'a', 'g', 't', 'g', 'g', 'c', 'a', 'a', 'g', 'a', 'c', '1280']\n",
      "['g', 'c', 'c', 't', 'g', 'a', 'c', 't', 'g', 't', 't', 't', 'a', 't', 'a', 't', 't', 't', 'g', 'c', '1280']\n",
      "['t', 't', 'a', 'a', 'a', 'c', 'c', 'c', 'g', 'a', 'c', 'g', 'c', 'a', 't', 'a', 'g', 'c', 'a', 'a', '1280']\n",
      "['c', 't', 'g', 'a', 't', 'g', 't', 't', 't', 't', 'g', 'c', 'c', 'g', 'c', 't', 'a', 't', 'a', 'c', '1280']\n"
     ]
    }
   ],
   "source": [
    "for each in fragments_toy.astype(\"str\").tolist()[:5]:\n",
    "    print(each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "409"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fragments_toy_rows = []\n",
    "for each in fragments_toy.astype(\"str\"):\n",
    "    row = concat_cols(each)\n",
    "    fragments_toy_rows.append(row)\n",
    "len(fragments_toy_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seq length: 1442\n",
      "(73, 21)\n",
      "All fragments found in this sequence\n",
      "\n",
      "seq length: 1917\n",
      "(96, 21)\n",
      "All fragments found in this sequence\n",
      "\n",
      "seq length: 1634\n",
      "(82, 21)\n",
      "All fragments found in this sequence\n",
      "\n",
      "seq length: 1840\n",
      "(92, 21)\n",
      "All fragments found in this sequence\n",
      "\n",
      "seq length: 1308\n",
      "(66, 21)\n",
      "All fragments found in this sequence\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mapping = {\"NC_013451\":b'1280',\n",
    "          'NC_006375':b'1590',\n",
    "           'NC_019565':b'210',\n",
    "           'NC_015407':b'2102',\n",
    "           'NC_016841':b'573'}\n",
    "\n",
    "input_file = \"data/train_small-db_toy-2000.fasta\"\n",
    "for seq_record in SeqIO.parse(input_file, \"fasta\"):\n",
    "    key = mapping[seq_record.id]\n",
    "    print(\"seq length:\", len(seq_record.seq))\n",
    "\n",
    "    fragments_to_consider = fragments_toy[np.where(fragments_toy[:,-1] == key)].astype(\"str\")\n",
    "    print(fragments_to_consider.shape)\n",
    "\n",
    "    for each in fragments_to_consider:\n",
    "        row = concat_cols(each)\n",
    "        assert row[0] in seq_record.seq, row[0]\n",
    "        assert row[1] == key.decode(\"utf-8\")\n",
    "    print(\"All fragments found in this sequence\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sampling2' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<timed exec>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'sampling2' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 1min 50sec\n",
    "\n",
    "seq_file = \"large-scale-metagenomics-1.0/data/train-dataset/train_small-db.fasta\"\n",
    "taxid_file = \"data/train_small-db.species-level.taxid\"\n",
    "output_dir = \"data/sampling/sampling-small-05-2\"\n",
    "sample_len = 200\n",
    "coverage = .05\n",
    "seed = 42\n",
    "\n",
    "sampling2.generate_fragment_data(seq_file, taxid_file, output_dir, sample_len, coverage, seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Encoding\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Toy Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(409, 921)\n",
      "(409,)\n"
     ]
    }
   ],
   "source": [
    "input_dir = \"data/sampling/sampling-toy-2000-2\"\n",
    "pattern = \"fragments*.npy\"\n",
    "\n",
    "fragments_toy2 = sampling2.read_fragments(input_dir, pattern)\n",
    "fragments_toy2.shape\n",
    "\n",
    "k=4\n",
    "X_toy_encoded2, y_toy2 = encoding2.encode_fragment_dataset(fragments_toy2, k)\n",
    "print(X_toy_encoded2.shape)\n",
    "print(y_toy2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "taxid_prob = naive_bayes.taxid_probability(y_toy2)\n",
    "print(taxid_prob)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Small Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(690539, 135168)\n",
      "Wall time: 1min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 1min 30sec\n",
    "\n",
    "input_dir = 'data/sampling/sampling-small-05-2'\n",
    "pattern = 'fragments*.npy'\n",
    "\n",
    "fragments_small2 = sampling2.read_fragments(input_dir, pattern)\n",
    "fragments_small2.shape\n",
    "\n",
    "k=6\n",
    "X_small_encoded2, y_small2 = encoding2.encode_fragment_dataset(fragments_small2, k)\n",
    "print(X_small_encoded2.shape)\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}