requirement_generator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "import textacy\n",
    "import numpy as np\n",
    "from sklearn.cluster import Birch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_colwidth = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_md')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_manager_reqs = pd.read_csv('./requirement/download_manager_reqs.csv')\n",
    "antivirus_reqs = pd.read_csv('./requirement/antivirus_reqs.csv')\n",
    "compression_reqs = pd.read_csv('./requirement/compression_reqs.csv')\n",
    "file_sharing_reqs = pd.read_csv('./requirement/file_sharing_reqs.csv')\n",
    "news_reqs = pd.read_csv('./requirement/news_reqs.csv')\n",
    "vpn_reqs = pd.read_csv('./requirement/vpn_reqs.csv')\n",
    "web_browser_reqs = pd.read_csv('./requirement/web_browser_reqs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "reqs_df = pd.concat([download_manager_reqs, antivirus_reqs, compression_reqs, file_sharing_reqs, news_reqs, vpn_reqs, web_browser_reqs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "reqs_df.drop(labels=['url', 'app'], axis=1, inplace=True)\n",
    "reqs_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add new column for requirement length\n",
    "reqs_df.loc[:, 'req_length'] = reqs_df['requirement'].apply(lambda r: len(r.split()))\n",
    "# add new column to store requirements as spacy documents\n",
    "reqs_df.loc[:, 'spacy_req'] = reqs_df['requirement'].apply(lambda r: nlp(r))\n",
    "# add new column for lemmatization doc\n",
    "reqs_df.loc[:, 'spacy_req_lemma'] = reqs_df['spacy_req'].apply(lambda doc: nlp(doc[:].lemma_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>requirement</th>\n",
       "      <th>req_length</th>\n",
       "      <th>spacy_req</th>\n",
       "      <th>spacy_req_lemma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Download Manager</td>\n",
       "      <td>Download accelerator</td>\n",
       "      <td>2</td>\n",
       "      <td>(Download, accelerator)</td>\n",
       "      <td>(download, accelerator)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Download Manager</td>\n",
       "      <td>Browser integration: IE, Chrome, Firefox, Opera</td>\n",
       "      <td>6</td>\n",
       "      <td>(Browser, integration, :, IE, ,, Chrome, ,, Fi...</td>\n",
       "      <td>(browser, integration, :, IE, ,, Chrome, ,, Fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Download Manager</td>\n",
       "      <td>Support for HTTP, HTTPS, FTP, MMS and RTSP</td>\n",
       "      <td>8</td>\n",
       "      <td>(Support, for, HTTP, ,, HTTPS, ,, FTP, ,, MMS,...</td>\n",
       "      <td>(support, for, HTTP, ,, HTTPS, ,, FTP, ,, MMS,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Download Manager</td>\n",
       "      <td>Multi-thread support to speed up downloads</td>\n",
       "      <td>6</td>\n",
       "      <td>(Multi, -, thread, support, to, speed, up, dow...</td>\n",
       "      <td>(multi, -, thread, support, to, speed, up, dow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Download Manager</td>\n",
       "      <td>Automatic or manual downloads</td>\n",
       "      <td>4</td>\n",
       "      <td>(Automatic, or, manual, downloads)</td>\n",
       "      <td>(automatic, or, manual, download)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             domain                                      requirement  \\\n",
       "0  Download Manager                             Download accelerator   \n",
       "1  Download Manager  Browser integration: IE, Chrome, Firefox, Opera   \n",
       "2  Download Manager       Support for HTTP, HTTPS, FTP, MMS and RTSP   \n",
       "3  Download Manager       Multi-thread support to speed up downloads   \n",
       "4  Download Manager                    Automatic or manual downloads   \n",
       "\n",
       "   req_length                                          spacy_req  \\\n",
       "0           2                            (Download, accelerator)   \n",
       "1           6  (Browser, integration, :, IE, ,, Chrome, ,, Fi...   \n",
       "2           8  (Support, for, HTTP, ,, HTTPS, ,, FTP, ,, MMS,...   \n",
       "3           6  (Multi, -, thread, support, to, speed, up, dow...   \n",
       "4           4                 (Automatic, or, manual, downloads)   \n",
       "\n",
       "                                     spacy_req_lemma  \n",
       "0                            (download, accelerator)  \n",
       "1  (browser, integration, :, IE, ,, Chrome, ,, Fi...  \n",
       "2  (support, for, HTTP, ,, HTTPS, ,, FTP, ,, MMS,...  \n",
       "3  (multi, -, thread, support, to, speed, up, dow...  \n",
       "4                  (automatic, or, manual, download)  "
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reqs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>requirement</th>\n",
       "      <th>req_length</th>\n",
       "      <th>spacy_req</th>\n",
       "      <th>spacy_req_lemma</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>domain</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Antivirus</th>\n",
       "      <td>1814</td>\n",
       "      <td>1814</td>\n",
       "      <td>1814</td>\n",
       "      <td>1814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Compression Tool</th>\n",
       "      <td>1327</td>\n",
       "      <td>1327</td>\n",
       "      <td>1327</td>\n",
       "      <td>1327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Download Manager</th>\n",
       "      <td>2775</td>\n",
       "      <td>2775</td>\n",
       "      <td>2775</td>\n",
       "      <td>2775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>File Sharing</th>\n",
       "      <td>1848</td>\n",
       "      <td>1848</td>\n",
       "      <td>1848</td>\n",
       "      <td>1848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>News</th>\n",
       "      <td>872</td>\n",
       "      <td>872</td>\n",
       "      <td>872</td>\n",
       "      <td>872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VPN</th>\n",
       "      <td>287</td>\n",
       "      <td>287</td>\n",
       "      <td>287</td>\n",
       "      <td>287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Web Browser</th>\n",
       "      <td>1756</td>\n",
       "      <td>1756</td>\n",
       "      <td>1756</td>\n",
       "      <td>1756</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  requirement  req_length  spacy_req  spacy_req_lemma\n",
       "domain                                                               \n",
       "Antivirus                1814        1814       1814             1814\n",
       "Compression Tool         1327        1327       1327             1327\n",
       "Download Manager         2775        2775       2775             2775\n",
       "File Sharing             1848        1848       1848             1848\n",
       "News                      872         872        872              872\n",
       "VPN                       287         287        287              287\n",
       "Web Browser              1756        1756       1756             1756"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reqs_df.groupby('domain').count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract boilerplate attributes given a spacy doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_boilerplate_attributes(doc):\n",
    "    verb_phrases = list(textacy.extract.pos_regex_matches(doc=doc, pattern=r'<VERB>?<ADV>*<VERB>+'))\n",
    "#     noun_phrases = list(doc.noun_chunks)\n",
    "    \n",
    "    verbs = []\n",
    "    objects = []\n",
    "    details = []\n",
    "    \n",
    "    for ix in range(len(verb_phrases)):\n",
    "        if ix < (len(verb_phrases) - 1):\n",
    "            internal_noun_phrases = list(doc[verb_phrases[ix].end:verb_phrases[ix+1].start].noun_chunks)\n",
    "        else:\n",
    "            internal_noun_phrases = list(doc[verb_phrases[ix].end:].noun_chunks)\n",
    "        \n",
    "        if (len(internal_noun_phrases) > 1):\n",
    "            objects.append(internal_noun_phrases[0].text)\n",
    "            details.append(internal_noun_phrases[1].text)\n",
    "            verbs.append(verb_phrases[ix].text)\n",
    "#     verbs = [v.text for v in verb_phrases]\n",
    "#     objects = [noun_phrases[i].text for i in range(len(noun_phrases)) if i % 2 == 0]\n",
    "#     details = [noun_phrases[i].text for i in range(len(noun_phrases)) if i % 2 == 1]\n",
    "    assert len(verbs) == len(objects) and len(objects) == len(details)\n",
    "    return verbs, objects, details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create boilerplate attributes dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boilerplate_df = pd.DataFrame(columns=['req_id', 'domain', 'verb', 'object', 'detail'])\n",
    "\n",
    "req_ids = []\n",
    "domains = []\n",
    "verbs = []\n",
    "objects = []\n",
    "additional_details = []\n",
    "\n",
    "for id, row in reqs_df.iterrows():\n",
    "#     print(row['spacy_req_lemma'])\n",
    "    vs, objs, dts = extract_boilerplate_attributes(row['spacy_req_lemma'])\n",
    "    if not vs or not objs or not dts:\n",
    "        continue\n",
    "#     print(vs, objs, dts)\n",
    "#     print(len(vs), len(objs), len(dts))\n",
    "    assert len(vs) == len(objs) and len(objs) == len(dts)\n",
    "    req_ids.extend([id] * len(vs))\n",
    "    domains.extend([row['domain']] * len(vs))\n",
    "    verbs.extend(vs)\n",
    "    objects.extend(objs)\n",
    "    additional_details.extend(dts)\n",
    "\n",
    "print(len(domains), len(req_ids),  len(additional_details), len(verbs), len(objects))\n",
    "    \n",
    "boilerplate_df = pd.DataFrame({\n",
    "    'req_id': req_ids,\n",
    "    'domain': domains,\n",
    "    'verb': verbs,\n",
    "    'object': objects,\n",
    "    'detail': additional_details\n",
    "})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Download Manager', 'Antivirus', 'Compression Tool',\n",
       "       'File Sharing', 'News', 'VPN', 'Web Browser'], dtype=object)"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boilerplate_df.domain.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[will make]"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list(textacy.extract.matches(doc=reqs_df['spacy_req_lemma'].loc[20], patterns=[{'POS': 'VERB'}, {'OP': '?'}, {'POS': 'ADV', 'OP': '*', 'POS': 'VERB', 'OP': '+'}]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "# construct boilerplate requirements and spacy requirements\n",
    "boilerplate_df['boilerplate_req'] = 'System shall provide the user with the ability to ' + boilerplate_df['verb'] + ' ' + boilerplate_df['object'] + ' ' + boilerplate_df['detail']\n",
    "boilerplate_df.loc[:, 'spacy_boilerplate_req'] = boilerplate_df['boilerplate_req'].apply(lambda r: nlp(r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>req_id</th>\n",
       "      <th>domain</th>\n",
       "      <th>verb</th>\n",
       "      <th>object</th>\n",
       "      <th>detail</th>\n",
       "      <th>boilerplate_req</th>\n",
       "      <th>spacy_boilerplate_req</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>Download Manager</td>\n",
       "      <td>check</td>\n",
       "      <td>file integrity</td>\n",
       "      <td>md5</td>\n",
       "      <td>System shall provide the user with the ability to check file integrity md5</td>\n",
       "      <td>(System, shall, provide, the, user, with, the, ability, to, check, file, integrity, md5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "      <td>Download Manager</td>\n",
       "      <td>support</td>\n",
       "      <td>all version</td>\n",
       "      <td>popular browser</td>\n",
       "      <td>System shall provide the user with the ability to support all version popular browser</td>\n",
       "      <td>(System, shall, provide, the, user, with, the, ability, to, support, all, version, popular, brow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>Download Manager</td>\n",
       "      <td>click</td>\n",
       "      <td>a download link</td>\n",
       "      <td>a browser</td>\n",
       "      <td>System shall provide the user with the ability to click a download link a browser</td>\n",
       "      <td>(System, shall, provide, the, user, with, the, ability, to, click, a, download, link, a, browser)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17</td>\n",
       "      <td>Download Manager</td>\n",
       "      <td>accelerate</td>\n",
       "      <td>IDM support HTTP</td>\n",
       "      <td>ftp</td>\n",
       "      <td>System shall provide the user with the ability to accelerate IDM support HTTP ftp</td>\n",
       "      <td>(System, shall, provide, the, user, with, the, ability, to, accelerate, IDM, support, HTTP, ftp)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18</td>\n",
       "      <td>Download Manager</td>\n",
       "      <td>can accelerate</td>\n",
       "      <td>download</td>\n",
       "      <td>up to 5 time</td>\n",
       "      <td>System shall provide the user with the ability to can accelerate download up to 5 time</td>\n",
       "      <td>(System, shall, provide, the, user, with, the, ability, to, can, accelerate, download, up, to, 5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   req_id            domain            verb            object  \\\n",
       "0       7  Download Manager           check    file integrity   \n",
       "1      16  Download Manager         support       all version   \n",
       "2      17  Download Manager           click   a download link   \n",
       "3      17  Download Manager      accelerate  IDM support HTTP   \n",
       "4      18  Download Manager  can accelerate          download   \n",
       "\n",
       "            detail  \\\n",
       "0              md5   \n",
       "1  popular browser   \n",
       "2        a browser   \n",
       "3              ftp   \n",
       "4     up to 5 time   \n",
       "\n",
       "                                                                          boilerplate_req  \\\n",
       "0              System shall provide the user with the ability to check file integrity md5   \n",
       "1   System shall provide the user with the ability to support all version popular browser   \n",
       "2       System shall provide the user with the ability to click a download link a browser   \n",
       "3       System shall provide the user with the ability to accelerate IDM support HTTP ftp   \n",
       "4  System shall provide the user with the ability to can accelerate download up to 5 time   \n",
       "\n",
       "                                                                                 spacy_boilerplate_req  \n",
       "0             (System, shall, provide, the, user, with, the, ability, to, check, file, integrity, md5)  \n",
       "1  (System, shall, provide, the, user, with, the, ability, to, support, all, version, popular, brow...  \n",
       "2    (System, shall, provide, the, user, with, the, ability, to, click, a, download, link, a, browser)  \n",
       "3     (System, shall, provide, the, user, with, the, ability, to, accelerate, IDM, support, HTTP, ftp)  \n",
       "4  (System, shall, provide, the, user, with, the, ability, to, can, accelerate, download, up, to, 5...  "
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boilerplate_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster and generate new requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cluster_requirements(df):\n",
    "    req_vectors = [req_vec.vector for req_vec in df['spacy_boilerplate_req']]\n",
    "    NUM_CLUSTERS = 5\n",
    "    brc = Birch(branching_factor=50, n_clusters=NUM_CLUSTERS, threshold=0.05, compute_labels=True)\n",
    "    brc.fit(req_vectors)\n",
    "    clusters = brc.predict(req_vectors)\n",
    "    # indexes[i] contains indices of all requirements belong to cluster (i+1)\n",
    "    indices = [np.where(clusters == i) for i in range(NUM_CLUSTERS)]\n",
    "    return indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOILERPLATE = 'System shall provide the user with the ability to {} {} {}'\n",
    "def make_requirements(elements):\n",
    "    \"\"\"\n",
    "    make a requirement from boilerplate elements\n",
    "    :param elements: DataFrame containing verb, object, and additional information of a requirement\n",
    "    :return: a list of requirements created by combining elements value\n",
    "    \"\"\"\n",
    "\n",
    "    verbs = []\n",
    "    objs = []\n",
    "    details = []\n",
    "\n",
    "    for req_series in elements:\n",
    "        verbs.append(req_series['verb'])\n",
    "        objs.append(req_series['object'])\n",
    "        details.append(req_series['detail'])\n",
    "\n",
    "    requirements = []\n",
    "    for i in range(len(verbs)):\n",
    "        for j in range(len(objs)):\n",
    "            for k in range(len(details)):\n",
    "                if i != j and j != k:\n",
    "                    requirements.append(make_requirement(verbs[i], objs[j], details[k]))\n",
    "\n",
    "    return requirements\n",
    "\n",
    "\n",
    "def make_requirement(verb, obj, detail):\n",
    "    return BOILERPLATE.format(verb, obj, detail)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Download Manager', 'Antivirus', 'Compression Tool',\n",
       "       'File Sharing', 'News', 'VPN', 'Web Browser'], dtype=object)"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boilerplate_df['domain'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_manager_df = boilerplate_df[boilerplate_df['domain'] == 'Download Manager']\n",
    "antivirus_df = boilerplate_df[boilerplate_df['domain'] == 'Antivirus']\n",
    "compression_tool_df = boilerplate_df[boilerplate_df['domain'] == 'Compression Tool']\n",
    "file_sharing_df = boilerplate_df[boilerplate_df['domain'] == 'File Sharing']\n",
    "news_df = boilerplate_df[boilerplate_df['domain'] == 'News']\n",
    "vpn_df = boilerplate_df[boilerplate_df['domain'] == 'VPN']\n",
    "web_browser_df = boilerplate_df[boilerplate_df['domain'] == 'Web Browser']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_manager_indices = cluster_requirements(download_manager_df)\n",
    "antivirus_indices = cluster_requirements(antivirus_df)\n",
    "compression_tool_indices = cluster_requirements(compression_tool_df)\n",
    "file_sharing_indices = cluster_requirements(file_sharing_df)\n",
    "news_indices = cluster_requirements(news_df)\n",
    "vpn_indices = cluster_requirements(vpn_df)\n",
    "web_browser_indices = cluster_requirements(web_browser_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_requirement(indices):\n",
    "    requirements = []\n",
    "    for ind in indices:\n",
    "        for _ in range(1):\n",
    "            # print(ind)\n",
    "            triple = np.random.choice(ind[0], 3)\n",
    "            bl_1 = boilerplate_df.iloc[triple[0]]\n",
    "            bl_2 = boilerplate_df.iloc[triple[1]]\n",
    "            bl_3 = boilerplate_df.iloc[triple[2]]\n",
    "            requirements.extend(make_requirements([bl_1, bl_2, bl_3]))\n",
    "#     print('----------------END-----------------\\n\\n\\n')\n",
    "    return pd.DataFrame({'requirement': requirements})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n",
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n",
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n",
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n",
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n",
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n",
      "----------------END-----------------\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "download_manager_requirements = generate_requirement(download_manager_indices)\n",
    "antivirus_requirements = generate_requirement(antivirus_indices)\n",
    "compression_tool_requirements = generate_requirement(compression_tool_indices)\n",
    "file_sharing_requirements = generate_requirement(file_sharing_indices)\n",
    "news_requirements = generate_requirement(news_indices)\n",
    "vpn_requirements = generate_requirement(vpn_indices)\n",
    "web_browser_requirements = generate_requirement(web_browser_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>requirement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>System shall provide the user with the ability to support such weakness medium file</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>System shall provide the user with the ability to support such weakness the express interface</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>System shall provide the user with the ability to support online movie medium file</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>System shall provide the user with the ability to support online movie the feature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>System shall provide the user with the ability to detect different format the feature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>System shall provide the user with the ability to detect different format the express interface</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>System shall provide the user with the ability to detect online movie medium file</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>System shall provide the user with the ability to detect online movie the feature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>System shall provide the user with the ability to can download different format the feature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>System shall provide the user with the ability to can download different format the express inte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>System shall provide the user with the ability to can download such weakness medium file</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>System shall provide the user with the ability to can download such weakness the express interface</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>System shall provide the user with the ability to download the ' hl ' tag both HTTP and ftp prot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>System shall provide the user with the ability to download the ' hl ' tag the avi format</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>System shall provide the user with the ability to download video both HTTP and ftp protocol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>System shall provide the user with the ability to download video google image search</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>System shall provide the user with the ability to set password protect site google image search</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>System shall provide the user with the ability to set password protect site the avi format</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>System shall provide the user with the ability to set video both HTTP and ftp protocol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>System shall provide the user with the ability to set video google image search</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>System shall provide the user with the ability to can easily save password protect site google i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>System shall provide the user with the ability to can easily save password protect site the avi ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>System shall provide the user with the ability to can easily save the ' hl ' tag both HTTP and f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>System shall provide the user with the ability to can easily save the ' hl ' tag the avi format</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>System shall provide the user with the ability to automatically parse the setting the video page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>System shall provide the user with the ability to automatically parse the setting -PRON- system</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>System shall provide the user with the ability to automatically parse Veoh the video page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>System shall provide the user with the ability to automatically parse Veoh -PRON- requirement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>System shall provide the user with the ability to adjust the address -PRON- requirement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>System shall provide the user with the ability to adjust the address -PRON- system</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>System shall provide the user with the ability to adjust Veoh the video page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>System shall provide the user with the ability to adjust Veoh -PRON- requirement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>System shall provide the user with the ability to may change the address -PRON- requirement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>System shall provide the user with the ability to may change the address -PRON- system</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>System shall provide the user with the ability to may change the setting the video page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>System shall provide the user with the ability to may change the setting -PRON- system</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>System shall provide the user with the ability to download any file simultaneous server connection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>System shall provide the user with the ability to download any file the world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>System shall provide the user with the ability to download the hot song simultaneous server conn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>System shall provide the user with the ability to download the hot song the entire archive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>System shall provide the user with the ability to extract the file the entire archive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>System shall provide the user with the ability to extract the file the world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>System shall provide the user with the ability to extract the hot song simultaneous server conne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>System shall provide the user with the ability to extract the hot song the entire archive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>System shall provide the user with the ability to always bring the file the entire archive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>System shall provide the user with the ability to always bring the file the world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>System shall provide the user with the ability to always bring any file simultaneous server conn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>System shall provide the user with the ability to always bring any file the world</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>System shall provide the user with the ability to show all download link the past web page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>System shall provide the user with the ability to show all download link download</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>System shall provide the user with the ability to show build - in calender the past web page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>System shall provide the user with the ability to show build - in calender the current page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>System shall provide the user with the ability to can add the history the current page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>System shall provide the user with the ability to can add the history download</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>System shall provide the user with the ability to can add build - in calender the past web page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>System shall provide the user with the ability to can add build - in calender the current page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>System shall provide the user with the ability to build the history the current page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>System shall provide the user with the ability to build the history download</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>System shall provide the user with the ability to build all download link the past web page</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>System shall provide the user with the ability to build all download link download</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                            requirement\n",
       "0                   System shall provide the user with the ability to support such weakness medium file\n",
       "1         System shall provide the user with the ability to support such weakness the express interface\n",
       "2                    System shall provide the user with the ability to support online movie medium file\n",
       "3                    System shall provide the user with the ability to support online movie the feature\n",
       "4                 System shall provide the user with the ability to detect different format the feature\n",
       "5       System shall provide the user with the ability to detect different format the express interface\n",
       "6                     System shall provide the user with the ability to detect online movie medium file\n",
       "7                     System shall provide the user with the ability to detect online movie the feature\n",
       "8           System shall provide the user with the ability to can download different format the feature\n",
       "9   System shall provide the user with the ability to can download different format the express inte...\n",
       "10             System shall provide the user with the ability to can download such weakness medium file\n",
       "11   System shall provide the user with the ability to can download such weakness the express interface\n",
       "12  System shall provide the user with the ability to download the ' hl ' tag both HTTP and ftp prot...\n",
       "13             System shall provide the user with the ability to download the ' hl ' tag the avi format\n",
       "14          System shall provide the user with the ability to download video both HTTP and ftp protocol\n",
       "15                 System shall provide the user with the ability to download video google image search\n",
       "16      System shall provide the user with the ability to set password protect site google image search\n",
       "17           System shall provide the user with the ability to set password protect site the avi format\n",
       "18               System shall provide the user with the ability to set video both HTTP and ftp protocol\n",
       "19                      System shall provide the user with the ability to set video google image search\n",
       "20  System shall provide the user with the ability to can easily save password protect site google i...\n",
       "21  System shall provide the user with the ability to can easily save password protect site the avi ...\n",
       "22  System shall provide the user with the ability to can easily save the ' hl ' tag both HTTP and f...\n",
       "23      System shall provide the user with the ability to can easily save the ' hl ' tag the avi format\n",
       "24     System shall provide the user with the ability to automatically parse the setting the video page\n",
       "25      System shall provide the user with the ability to automatically parse the setting -PRON- system\n",
       "26            System shall provide the user with the ability to automatically parse Veoh the video page\n",
       "27        System shall provide the user with the ability to automatically parse Veoh -PRON- requirement\n",
       "28              System shall provide the user with the ability to adjust the address -PRON- requirement\n",
       "29                   System shall provide the user with the ability to adjust the address -PRON- system\n",
       "30                         System shall provide the user with the ability to adjust Veoh the video page\n",
       "31                     System shall provide the user with the ability to adjust Veoh -PRON- requirement\n",
       "32          System shall provide the user with the ability to may change the address -PRON- requirement\n",
       "33               System shall provide the user with the ability to may change the address -PRON- system\n",
       "34              System shall provide the user with the ability to may change the setting the video page\n",
       "35               System shall provide the user with the ability to may change the setting -PRON- system\n",
       "36   System shall provide the user with the ability to download any file simultaneous server connection\n",
       "37                        System shall provide the user with the ability to download any file the world\n",
       "38  System shall provide the user with the ability to download the hot song simultaneous server conn...\n",
       "39           System shall provide the user with the ability to download the hot song the entire archive\n",
       "40                System shall provide the user with the ability to extract the file the entire archive\n",
       "41                         System shall provide the user with the ability to extract the file the world\n",
       "42  System shall provide the user with the ability to extract the hot song simultaneous server conne...\n",
       "43            System shall provide the user with the ability to extract the hot song the entire archive\n",
       "44           System shall provide the user with the ability to always bring the file the entire archive\n",
       "45                    System shall provide the user with the ability to always bring the file the world\n",
       "46  System shall provide the user with the ability to always bring any file simultaneous server conn...\n",
       "47                    System shall provide the user with the ability to always bring any file the world\n",
       "48           System shall provide the user with the ability to show all download link the past web page\n",
       "49                    System shall provide the user with the ability to show all download link download\n",
       "50         System shall provide the user with the ability to show build - in calender the past web page\n",
       "51          System shall provide the user with the ability to show build - in calender the current page\n",
       "52               System shall provide the user with the ability to can add the history the current page\n",
       "53                       System shall provide the user with the ability to can add the history download\n",
       "54      System shall provide the user with the ability to can add build - in calender the past web page\n",
       "55       System shall provide the user with the ability to can add build - in calender the current page\n",
       "56                 System shall provide the user with the ability to build the history the current page\n",
       "57                         System shall provide the user with the ability to build the history download\n",
       "58          System shall provide the user with the ability to build all download link the past web page\n",
       "59                   System shall provide the user with the ability to build all download link download"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "antivirus_requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}