Workbook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import requests, random, time\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlparse, urljoin\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get base links\n",
    "repos = [\n",
    "    \"html-css\"\n",
    "    \"ethics\",\n",
    "    \"python\",\n",
    "    \"machine-learning\",\n",
    "    \"text-analysis\",\n",
    "    \"command-line\",\n",
    "    \"twitter-api\",\n",
    "    \"omeka\",\n",
    "    \"databases\",\n",
    "    \"project-lab\",\n",
    "    \"git\",\n",
    "    \"mapping\"\n",
    "]\n",
    "\n",
    "\n",
    "to_visit, outlinks = [], []\n",
    "\n",
    "for repo in repos:\n",
    "    site = f\"https://www.github.com/DHRI-Curriculum/{repo}\"\n",
    "    to_visit.append(site)\n",
    "\n",
    "base = urlparse(site).netloc\n",
    "visited, external_visited = {}, {}\n",
    "\n",
    "s404 = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "225/0\n"
     ]
    }
   ],
   "source": [
    "previous = None\n",
    "i = 1\n",
    "while to_visit:\n",
    "    l = to_visit.pop()\n",
    "\n",
    "    if l.split(\".\")[-1].lower() in ['png', 'jpg', 'gif', 'ipynb', 'csv', 'jpeg']: continue # skip images and other formats\n",
    "    \n",
    "    clear_output(wait=True)\n",
    "    print(f\"{i}/{len(to_visit)}\")\n",
    "    i+=1\n",
    "    \n",
    "    '''\n",
    "    _wait = random.randint(5,15)\n",
    "    print(f\"Waiting for {_wait}...\")\n",
    "    time.sleep(_wait)\n",
    "    '''\n",
    "    #print(l)\n",
    "    url = urljoin(site, l)\n",
    "\n",
    "    try:\n",
    "        r = requests.get(url)\n",
    "        visited[l] = r.status_code\n",
    "\n",
    "    except:\n",
    "        visited[l] = None\n",
    "\n",
    "    if r.status_code == 429:\n",
    "        raise RuntimeError(\"Reached 429...\")\n",
    "        \n",
    "    if r.status_code == 404:\n",
    "        print(f\"404 found in {previous}: {r.url}\")\n",
    "        s404.append({'source': previous, 'target': r.url})\n",
    "    \n",
    "    if r.status_code == 200:\n",
    "        soup = BeautifulSoup(r.content, 'html5lib')\n",
    "        article = soup.find('article')\n",
    "        try:\n",
    "            links = [l['href'] for l in article.find_all('a', href=True)]\n",
    "        except AttributeError:\n",
    "            links = []\n",
    "            print(f\"An error occurred here - no 'article' element found in {r.url}.\")\n",
    "        for link in links:\n",
    "            parsed_link = urlparse(link)\n",
    "            loc = parsed_link.netloc\n",
    "            path = parsed_link.path\n",
    "            joined_url = urljoin(site, link)\n",
    "\n",
    "            if loc == '':\n",
    "                if joined_url not in to_visit and joined_url not in visited.keys():\n",
    "                    to_visit.append(joined_url)\n",
    "\n",
    "            elif loc == base:\n",
    "                if link not in to_visit and link not in visited.keys():\n",
    "                    to_visit.append(link)\n",
    "\n",
    "            else:\n",
    "                if link not in outlinks and link not in visited.keys():\n",
    "                    outlinks.append(link)\n",
    "    \n",
    "    to_visit = list(set([x.split('#')[0] for x in to_visit])) # remove any anchors\n",
    "    to_visit = [x for x in to_visit if x not in visited.keys()]\n",
    "    \n",
    "    previous = l\n",
    "    \n",
    "i = 1\n",
    "\n",
    "# check the status of external links\n",
    "while outlinks:\n",
    "    l = outlinks.pop()\n",
    "    \n",
    "    clear_output(wait=True)\n",
    "    print(f\"{i}/{len(to_visit)}\")\n",
    "    i+=1\n",
    "\n",
    "    try:\n",
    "        r = requests.get(l)\n",
    "        external_visited[l] = r.status_code\n",
    "\n",
    "    except:\n",
    "        external_visited[l] = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "s = pd.Series(visited, name='Response')\n",
    "s.index.name = 'URL'\n",
    "df1 = pd.DataFrame(s)\n",
    "df1['Type'] = 'Internal'\n",
    "\n",
    "s = pd.Series(external_visited, name='Response')\n",
    "s.index.name = 'URL'\n",
    "df2 = pd.DataFrame(s)\n",
    "df2['Type'] = 'External'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Response</th>\n",
       "      <th>Type</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>URL</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/mapping</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/mapping/blob/master/sections/15calc.md</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/mapping/blob/master/sections/19layout.md</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/python/blob/master/sections/06-loops.md</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/python/blob/master/sections/solutions/loop.md</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/command-line/blob/master/sections/11-summary.md</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/omeka/blob/master/sections/theoreticalconcerns.md</th>\n",
       "      <td>200</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/omeka/blob/master/sections/.../README.md</th>\n",
       "      <td>404</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/machine-learning/blob/master/sections/07-supervised.md</th>\n",
       "      <td>404</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://www.github.com/DHRI-Curriculum/html-cssethics</th>\n",
       "      <td>404</td>\n",
       "      <td>Internal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>166 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Response      Type\n",
       "URL                                                                   \n",
       "https://www.github.com/DHRI-Curriculum/mapping           200  Internal\n",
       "https://www.github.com/DHRI-Curriculum/mapping/...       200  Internal\n",
       "https://www.github.com/DHRI-Curriculum/mapping/...       200  Internal\n",
       "https://www.github.com/DHRI-Curriculum/python/b...       200  Internal\n",
       "https://www.github.com/DHRI-Curriculum/python/b...       200  Internal\n",
       "...                                                      ...       ...\n",
       "https://www.github.com/DHRI-Curriculum/command-...       200  Internal\n",
       "https://www.github.com/DHRI-Curriculum/omeka/bl...       200  Internal\n",
       "https://www.github.com/DHRI-Curriculum/omeka/bl...       404  Internal\n",
       "https://www.github.com/DHRI-Curriculum/machine-...       404  Internal\n",
       "https://www.github.com/DHRI-Curriculum/html-css...       404  Internal\n",
       "\n",
       "[166 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.sort_values(\"Response\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Response</th>\n",
       "      <th>Type</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>URL</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>http://www.loc.gov/preservation/resources/rfs/TOC.html</th>\n",
       "      <td>200.0</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>http://www.todayifoundout.com/index.php/2015/04/c-drive-default-windows-based-computers-2/</th>\n",
       "      <td>200.0</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://genius.com/Johanna-drucker-from-a-to-screen-annotated</th>\n",
       "      <td>200.0</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://github.com/DHRI-Curriculum/glossary/blob/master/sections/command-line.md</th>\n",
       "      <td>200.0</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://youtu.be/Xq29uTtKW4M</th>\n",
       "      <td>200.0</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://de.dariah.eu/tatom/index.html</th>\n",
       "      <td>NaN</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>http://www.conf.org/whoweserve/community/resources/index.cfm?navitemNumber=15626#locator</th>\n",
       "      <td>NaN</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>http://dirtdirectory.org/</th>\n",
       "      <td>NaN</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://anh.cs.luc.edu/python/hands-on/3.1/handsonHtml/index.html</th>\n",
       "      <td>NaN</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>https://academic.oup.com/dsh/article-abstract/28/2/279/1037192?redirectedFrom=fulltext</th>\n",
       "      <td>NaN</td>\n",
       "      <td>External</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>225 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Response      Type\n",
       "URL                                                                   \n",
       "http://www.loc.gov/preservation/resources/rfs/T...     200.0  External\n",
       "http://www.todayifoundout.com/index.php/2015/04...     200.0  External\n",
       "https://genius.com/Johanna-drucker-from-a-to-sc...     200.0  External\n",
       "https://github.com/DHRI-Curriculum/glossary/blo...     200.0  External\n",
       "https://youtu.be/Xq29uTtKW4M                           200.0  External\n",
       "...                                                      ...       ...\n",
       "https://de.dariah.eu/tatom/index.html                    NaN  External\n",
       "http://www.conf.org/whoweserve/community/resour...       NaN  External\n",
       "http://dirtdirectory.org/                                NaN  External\n",
       "https://anh.cs.luc.edu/python/hands-on/3.1/hand...       NaN  External\n",
       "https://academic.oup.com/dsh/article-abstract/2...       NaN  External\n",
       "\n",
       "[225 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.sort_values(\"Response\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'source': 'https://www.github.com/DHRI-Curriculum/mapping/blob/master/sections/4layerpr.md',\n",
       "  'target': 'https://github.com/DHRI-Curriculum/html-cssethics'},\n",
       " {'source': 'https://www.github.com/DHRI-Curriculum/omeka/blob/master/sections/themesplugins.md',\n",
       "  'target': 'https://github.com/DHRI-Curriculum/machine-learning/blob/master/sections/07-supervised.md'},\n",
       " {'source': 'https://www.github.com/DHRI-Curriculum/command-line/blob/master/sections/09-data.md',\n",
       "  'target': 'https://github.com/DHRI-Curriculum/omeka/blob/master/sections/.../README.md'}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s404"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.0 64-bit",
   "language": "python",
   "name": "python38064bitd74070990f3d402eb483695c8dddeff7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}