13F.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "8bc58491",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "03e871fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes of each table: \n",
      "['table', 'table-hover', 'entity-hints']\n",
      "['table']\n"
     ]
    }
   ],
   "source": [
    "page = requests.get('https://www.sec.gov/edgar/search/#/q=13-f')\n",
    "soup = BeautifulSoup(page.text, 'html.parser')\n",
    "#soup.find('td', text='13F-HR (Institutional investment manager holdings report) ').find_next_sibling('td').text\n",
    "\n",
    "# verifying tables and their classes\n",
    "print('Classes of each table: ')\n",
    "for table in soup.find_all('table'):\n",
    "    print(table.get('class'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "6f430ae0",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'urlopen' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [73]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m html \u001b[38;5;241m=\u001b[39m \u001b[43murlopen\u001b[49m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://www.sec.gov/edgar/search/#/q=13-f\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      2\u001b[0m soup \u001b[38;5;241m=\u001b[39m BeautifulSoup(html\u001b[38;5;241m.\u001b[39mread(), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlxml\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'urlopen' is not defined"
     ]
    }
   ],
   "source": [
    "html = urlopen('https://www.sec.gov/edgar/search/#/q=13-f')\n",
    "soup = BeautifulSoup(html.read(), 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "10804776",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Input \u001b[0;32mIn [70]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m row \u001b[38;5;129;01min\u001b[39;00m soup\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mth\u001b[39m\u001b[38;5;124m'\u001b[39m)[\u001b[38;5;241m1\u001b[39m:]:\n\u001b[0;32m----> 2\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[43mrow\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'text'"
     ]
    }
   ],
   "source": [
    "for row in soup.find_all('t')[1:]:\n",
    "    print(row.td.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "c36b57f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Empty DataFrame\n",
       " Columns: [Form & File, Filed, Reporting for, Filing entity/person, CIK, Located, Incorporated, File number, Film number]\n",
       " Index: []]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page = requests.get(url)\n",
    "soup = BeautifulSoup(page.text, 'html.parser')\n",
    "\n",
    "dfs = pd.read_html(page.text)\n",
    "dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "888ebcf7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tr>\n",
       " <th class=\"filetype\" id=\"filetype\">Form &amp; File</th>\n",
       " <th class=\"filed\">Filed</th>\n",
       " <th class=\"enddate\">Reporting for</th>\n",
       " <th class=\"entity-name\">Filing entity/person</th>\n",
       " <th class=\"cik\">CIK</th>\n",
       " <th class=\"located\">Located</th>\n",
       " <th class=\"incorporated\">Incorporated</th>\n",
       " <th class=\"file-num\">File number</th>\n",
       " <th class=\"film-num\">Film number</th>\n",
       " </tr>]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag = soup.findAll('tr')\n",
    "tag_tr = tag[0].find('th')\n",
    "tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "6e68fab7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tr>\n",
       " <th class=\"filetype\" id=\"filetype\">Form &amp; File</th>\n",
       " <th class=\"filed\">Filed</th>\n",
       " <th class=\"enddate\">Reporting for</th>\n",
       " <th class=\"entity-name\">Filing entity/person</th>\n",
       " <th class=\"cik\">CIK</th>\n",
       " <th class=\"located\">Located</th>\n",
       " <th class=\"incorporated\">Incorporated</th>\n",
       " <th class=\"file-num\">File number</th>\n",
       " <th class=\"film-num\">Film number</th>\n",
       " </tr>]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tables = soup.findAll('table')[1].findAll('tr')\n",
    "tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "5d6d9700",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'function' object has no attribute 'next'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Input \u001b[0;32mIn [98]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m row_list \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m      4\u001b[0m row_list\u001b[38;5;241m.\u001b[39mappend(item\u001b[38;5;241m.\u001b[39mtext\u001b[38;5;241m.\u001b[39mstrip())\n\u001b[0;32m----> 5\u001b[0m siblings \u001b[38;5;241m=\u001b[39m \u001b[43mitem\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfind_all\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnext\u001b[49m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtr\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m sibling \u001b[38;5;129;01min\u001b[39;00m siblings:\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFiling entity/person\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m siblings\u001b[38;5;241m.\u001b[39mtext:\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'next'"
     ]
    }
   ],
   "source": [
    "final_list = []\n",
    "for item in soup.findAll('table')[1].findAll('th', text=re.compile(\"Filing entity/person\")):\n",
    "    row_list = []\n",
    "    row_list.append(item.text.strip())\n",
    "    siblings = item.find_all.next('tr')\n",
    "    for sibling in siblings:\n",
    "        if \"Filing entity/person\" in siblings.text:\n",
    "            break\n",
    "        else:\n",
    "            if sibling.text.strip()!='':\n",
    "                row_list.append(sibling.text.strip())\n",
    "    final_list.append(row_list)\n",
    "final_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d9e29dac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Empty DataFrame\n",
       " Columns: [Form & File, Filed, Reporting for, Filing entity/person, CIK, Located, Incorporated, File number, Film number]\n",
       " Index: []]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://www.sec.gov/edgar/search/#/q=13-f'\n",
    "dfs = pd.read_html(url)\n",
    "dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cf8d441e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sec_url = 'https://www.sec.gov'\n",
    "\n",
    "def get_request(url):\n",
    "    headers = {\n",
    "        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36',\n",
    "        'Accept-Encoding': 'gzip, deflate, br',\n",
    "        'HOST': 'www.sec.gov',\n",
    "    }\n",
    "    return requests.get(url, headers=headers)\n",
    "\n",
    "def create_url(cik):\n",
    "    return 'https://www.sec.gov/cgi-bin/browse-edgar?CIK={}&owner=exclude&action=getcompany&type=13F-HR'.format(cik)\n",
    "\n",
    "def get_user_input():\n",
    "    cik = input(\"Enter 10-digit CIK number: \")\n",
    "    return cik"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d931ffc7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}