6.news_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/barry/miniconda3/envs/bertenv2/lib/python3.7/site-packages/tqdm/std.py:648: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 匯入資料"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1-1 匯入危機公司新聞"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-1-1 匯入TEJ危機公司新聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3444, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>公司碼</th>\n",
       "      <th>年月</th>\n",
       "      <th>則次</th>\n",
       "      <th>序號</th>\n",
       "      <th>新聞內容</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2474 可成</td>\n",
       "      <td>2017/12/29</td>\n",
       "      <td>01</td>\n",
       "      <td>10</td>\n",
       "      <td>可成旗下子公司可成科技( 宿遷) 及可功科技( 宿遷)28 日分別斥資6.2 億元及6.3 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1611 中電</td>\n",
       "      <td>2017/12/28</td>\n",
       "      <td>01</td>\n",
       "      <td>10</td>\n",
       "      <td>公開資訊觀測站重大訊息公告                                 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       公司碼          年月  則次  序號  \\\n",
       "0  2474 可成  2017/12/29  01  10   \n",
       "1  1611 中電  2017/12/28  01  10   \n",
       "\n",
       "                                                新聞內容  \n",
       "0  可成旗下子公司可成科技( 宿遷) 及可功科技( 宿遷)28 日分別斥資6.2 億元及6.3 ...  \n",
       "1  公開資訊觀測站重大訊息公告                                 ...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tej_distress = pd.read_csv('data/2018_2017_tej_distress_0423.csv')\n",
    "print(tej_distress.shape)\n",
    "tej_distress.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-1-2 匯入UDN危機公司新聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 設定路徑，將該資料夾內所有檔案名稱列出來\n",
    "Folder_Path = 'data/udn_distress/'\n",
    "file_list = os.listdir(Folder_Path)\n",
    "# file_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2589, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>time</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>【2009-04-13/經濟日報/C3版/市場焦點】</td>\n",
       "      <td>（\\r\\n）積極搶攻電信業自有品牌小筆電市場宣告開花結果。\\r\\n主管表示，\\r\\n近期拿下...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>【2009-05-08/經濟日報/C3版/市場焦點】</td>\n",
       "      <td>（\\r\\n）營運出現大轉機，新任總經理許志宏昨（7）日上任，\\r\\n將再搶下大陸第二階段家電...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company                        time  \\\n",
       "0  2331 精英  【2009-04-13/經濟日報/C3版/市場焦點】   \n",
       "1  2331 精英  【2009-05-08/經濟日報/C3版/市場焦點】   \n",
       "\n",
       "                                             content  \n",
       "0  （\\r\\n）積極搶攻電信業自有品牌小筆電市場宣告開花結果。\\r\\n主管表示，\\r\\n近期拿下...  \n",
       "1  （\\r\\n）營運出現大轉機，新任總經理許志宏昨（7）日上任，\\r\\n將再搶下大陸第二階段家電...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udn_distress = pd.DataFrame(columns=['company', 'title', 'time', 'content'])\n",
    "\n",
    "for file in file_list:\n",
    "    df_temp = pd.read_csv('data/udn_distress/' + file)\n",
    "    udn_distress = pd.concat([udn_distress, df_temp])\n",
    "udn_distress = udn_distress.drop('title', axis=1).dropna()\n",
    "print(udn_distress.shape)\n",
    "udn_distress.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-1-3 （針對UDN）擷取日期，把日期格式轉換為yyyy/mm/dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "【2009-05-25/經濟日報/C3版/市場焦點】\n",
      "2009/05/25\n",
      "2009-05-25\n"
     ]
    }
   ],
   "source": [
    "from datetime import date\n",
    "import datetime\n",
    "\n",
    "#測試一下\n",
    "\n",
    "s = udn_distress['time'].iloc[4]\n",
    "print(s)\n",
    "match = re.search('\\d+-\\d+-\\d+', s)\n",
    "date = str(match.group(0))\n",
    "date = date.replace('-', '/')\n",
    "print(date)\n",
    "date = datetime.datetime.strptime(date, '%Y/%m/%d').date()\n",
    "# date = datetime.datetime.strptime(match.group(0), '%Y-%m-%d').date()\n",
    "print(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2589, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>time</th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>【2009-04-13/經濟日報/C3版/市場焦點】</td>\n",
       "      <td>（\\r\\n）積極搶攻電信業自有品牌小筆電市場宣告開花結果。\\r\\n主管表示，\\r\\n近期拿下...</td>\n",
       "      <td>2009/04/13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>【2009-05-08/經濟日報/C3版/市場焦點】</td>\n",
       "      <td>（\\r\\n）營運出現大轉機，新任總經理許志宏昨（7）日上任，\\r\\n將再搶下大陸第二階段家電...</td>\n",
       "      <td>2009/05/08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company                        time  \\\n",
       "0  2331 精英  【2009-04-13/經濟日報/C3版/市場焦點】   \n",
       "1  2331 精英  【2009-05-08/經濟日報/C3版/市場焦點】   \n",
       "\n",
       "                                             content        date  \n",
       "0  （\\r\\n）積極搶攻電信業自有品牌小筆電市場宣告開花結果。\\r\\n主管表示，\\r\\n近期拿下...  2009/04/13  \n",
       "1  （\\r\\n）營運出現大轉機，新任總經理許志宏昨（7）日上任，\\r\\n將再搶下大陸第二階段家電...  2009/05/08  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dates = []\n",
    "for time in udn_distress['time']:\n",
    "    s = time\n",
    "    match = re.search('\\d+-\\d+-\\d+', s)\n",
    "    date = str(match.group(0))\n",
    "    date = date.replace('-', '/')\n",
    "    dates.append(date)\n",
    "udn_distress['date'] = dates\n",
    "# udn_distress.drop('time', axis=1, inplace=True)\n",
    "print(udn_distress.shape)\n",
    "udn_distress.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2589, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>time</th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>【2009-04-13/經濟日報/C3版/市場焦點】</td>\n",
       "      <td>（  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...</td>\n",
       "      <td>2009/04/13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>【2009-05-08/經濟日報/C3版/市場焦點】</td>\n",
       "      <td>（  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...</td>\n",
       "      <td>2009/05/08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company                        time  \\\n",
       "0  2331 精英  【2009-04-13/經濟日報/C3版/市場焦點】   \n",
       "1  2331 精英  【2009-05-08/經濟日報/C3版/市場焦點】   \n",
       "\n",
       "                                             content        date  \n",
       "0  （  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...  2009/04/13  \n",
       "1  （  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...  2009/05/08  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 把一些字串刪掉改空格\n",
    "udn_distress['content'] = udn_distress['content'].astype(str).str.replace('\\r' , ' ').str.replace('\\t' , ' ').str.replace(\"\\n\", \" \").str.replace(\",\", \" \").str.replace('\"' , ' ').str.replace(\"'\" , ' ')\n",
    "udn_distress['company'] = udn_distress['company'].astype(str).str.replace('\\r' , ' ').str.replace('\\t' , ' ').str.replace(\"\\n\", \" \").str.replace(\",\", \" \").str.replace('\"' , ' ').str.replace(\"'\" , ' ')\n",
    "print(udn_distress.shape)\n",
    "udn_distress.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2589, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>（  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...</td>\n",
       "      <td>2009/04/13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>（  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...</td>\n",
       "      <td>2009/05/08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company                                            content        date\n",
       "0  2331 精英  （  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...  2009/04/13\n",
       "1  2331 精英  （  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...  2009/05/08"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udn_distress = udn_distress.loc[:, ['company', 'content', 'date']]\n",
    "print(udn_distress.shape)\n",
    "udn_distress.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "udn_distress.to_csv(\"data/udn_distress_news.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2589, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>（  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...</td>\n",
       "      <td>2009/04/13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>（  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...</td>\n",
       "      <td>2009/05/08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company                                            content        date\n",
       "0  2331 精英  （  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...  2009/04/13\n",
       "1  2331 精英  （  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...  2009/05/08"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "udn_distress = pd.read_csv(\"data/udn_distress_news.csv\", encoding=\"utf-8\")\n",
    "print(udn_distress.shape)\n",
    "udn_distress.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-1-4 合併危機公司資料集為 distress_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6033, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>2009/04/13</td>\n",
       "      <td>（  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>2009/05/08</td>\n",
       "      <td>（  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company        date                                            content\n",
       "0  2331 精英  2009/04/13  （  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...\n",
       "1  2331 精英  2009/05/08  （  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distress_news = pd.DataFrame(columns=['company', 'date', 'content'])\n",
    "company = udn_distress['company']\n",
    "company = company.append(tej_distress['公司碼'])\n",
    "date = udn_distress['date']\n",
    "date = date.append(tej_distress['年月'])\n",
    "content = udn_distress['content']\n",
    "content = content.append(tej_distress['新聞內容'])\n",
    "distress_news['company'] = company\n",
    "distress_news['date'] = date\n",
    "distress_news['content'] = content\n",
    "print(distress_news.shape)\n",
    "distress_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distress_news.to_csv(\"data/tej_udn_distress_news.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6033, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>2009/04/13</td>\n",
       "      <td>（  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2331 精英</td>\n",
       "      <td>2009/05/08</td>\n",
       "      <td>（  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company        date                                            content\n",
       "0  2331 精英  2009/04/13  （  ）積極搶攻電信業自有品牌小筆電市場宣告開花結果。  主管表示，  近期拿下遠傳（490...\n",
       "1  2331 精英  2009/05/08  （  ）營運出現大轉機，新任總經理許志宏昨（7）日上任，  將再搶下大陸第二階段家電下鄉大單..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distress_news = pd.read_csv(\"data/tej_udn_distress_news.csv\", encoding=\"utf-8\")\n",
    "print(distress_news.shape)\n",
    "distress_news.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1-2 匯入正常公司新聞"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-2-1 匯入TFJ正常公司新聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7269, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>公司碼</th>\n",
       "      <th>年月</th>\n",
       "      <th>則次</th>\n",
       "      <th>序號</th>\n",
       "      <th>新聞內容</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017/12/26</td>\n",
       "      <td>01</td>\n",
       "      <td>10</td>\n",
       "      <td>三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2049 上銀</td>\n",
       "      <td>2017/12/11</td>\n",
       "      <td>01</td>\n",
       "      <td>10</td>\n",
       "      <td>上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         公司碼          年月  則次  序號  \\\n",
       "0  9946 三發地產  2017/12/26  01  10   \n",
       "1    2049 上銀  2017/12/11  01  10   \n",
       "\n",
       "                                                新聞內容  \n",
       "0  三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...  \n",
       "1  上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tej_normal = pd.read_csv('data/2018_2017_tej_normal_0423.csv')\n",
    "print(tej_normal.shape)\n",
    "tej_normal.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-2-2 匯入UDN正常公司新聞（0422：正常公司的UDN資料蒐集不完整，暫不採納）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Folder_Path = '/home/michael/Documents/M10609101_Thesis/News/udn/normal/'\n",
    "# file_list = os.listdir(Folder_Path)\n",
    "# # file_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# udn_normal = pd.DataFrame(columns=['company', 'title', 'time', 'content'])\n",
    "\n",
    "# for file in file_list:\n",
    "#     df_temp = pd.read_csv('News/udn/normal/' + file)\n",
    "#     udn_normal = pd.concat([udn_normal, df_temp])\n",
    "# udn_normal = udn_normal.drop('title', axis=1).dropna()\n",
    "# udn_normal.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(udn_normal)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-2-3 （針對UDN）擷取日期，把日期格式轉換為yyyy/mm/dd（0422：正常公司的UDN資料蒐集不完整，暫不採納）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dates = []\n",
    "# for time in udn_normal['time']:\n",
    "#     s = time\n",
    "#     match = re.search('\\d+-\\d+-\\d+', s)\n",
    "#     date = str(match.group(0))\n",
    "#     date = date.replace('-', '/')\n",
    "#     dates.append(date)\n",
    "# udn_normal['date'] = dates\n",
    "# # udn_normal.drop('time', axis=1, inplace=True)\n",
    "# udn_normal.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-2-4 合併危機公司資料集為 normal_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7269, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017/12/26</td>\n",
       "      <td>三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2049 上銀</td>\n",
       "      <td>2017/12/11</td>\n",
       "      <td>上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     company        date                                            content\n",
       "0  9946 三發地產  2017/12/26  三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...\n",
       "1    2049 上銀  2017/12/11  上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_news = pd.DataFrame(columns=['company', 'date', 'content'])\n",
    "normal_news['company'] = tej_normal['公司碼']\n",
    "normal_news['date'] = tej_normal['年月']\n",
    "normal_news['content'] = tej_normal['新聞內容']\n",
    "print(normal_news.shape)\n",
    "normal_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# normal_news = pd.DataFrame(columns=['company', 'date', 'content'])\n",
    "# company = udn_normal['company']\n",
    "# company = company.append(tej_normal['公司碼'])\n",
    "# date = udn_normal['date']\n",
    "# date = date.append(tej_normal['年月'])\n",
    "# content = udn_normal['content']\n",
    "# content = content.append(tej_normal['新聞內容'])\n",
    "# normal_news['company'] = company\n",
    "# normal_news['date'] = date\n",
    "# normal_news['content'] = content\n",
    "# normal_news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normal_news.to_csv(\"data/tej_normal_news.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7269, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017/12/26</td>\n",
       "      <td>三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2049 上銀</td>\n",
       "      <td>2017/12/11</td>\n",
       "      <td>上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     company        date                                            content\n",
       "0  9946 三發地產  2017/12/26  三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...\n",
       "1    2049 上銀  2017/12/11  上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_news = pd.read_csv(\"data/tej_normal_news.csv\", encoding=\"utf-8\")\n",
    "print(normal_news.shape)\n",
    "normal_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017/12/26</td>\n",
       "      <td>三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2049 上銀</td>\n",
       "      <td>2017/12/11</td>\n",
       "      <td>上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     company        date                                            content\n",
       "0  9946 三發地產  2017/12/26  三發地產(9946)昨(25)日舉行法說會時表示，總銷50億元的高雄「三發晶沙」、總銷50億...\n",
       "1    2049 上銀  2017/12/11  上銀(2049)公佈11月合併營收首度突破20億元，達20.41億元，月增3.5%、年增34..."
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news = pd.concat([normal_news, distress_news])   # 上下合併\n",
    "print(all_news.shape)\n",
    "all_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5469 瀚宇博</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    company       date                                            content\n",
       "0  2316 楠梓電 2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...\n",
       "1  5469 瀚宇博 2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機..."
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news['date'] = pd.to_datetime(all_news['date'])\n",
    "all_news = all_news.sort_values(by=[\"date\",\"company\"]).reset_index(drop=True) # 用post_time排序 # 在重設index\n",
    "print(all_news.shape)\n",
    "all_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_news.to_csv(\"data/tej_udn_all_news.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-3 每間公司風險分布\n",
    "使用意藍2015年風險資料集、經過BERT主題模型產生的model 將每篇文章輸出1*5的output\n",
    "\n",
    "13302篇文章就有13302*5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/barry/miniconda3/envs/bertenv/lib/python3.7/site-packages/tqdm/std.py:648: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, TFBertModel, TFBertForSequenceClassification, AdamW\n",
    "\n",
    "from tensorflow.keras import Sequential, Model, Input\n",
    "from tensorflow.keras.layers import Dense, Embedding, SpatialDropout1D, Dropout, Activation, Flatten, InputLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>台灣表面黏著科技（6278）受惠台灣近年面板蓬勃發展，去年成為最大的印刷電路板表面黏著科技廠...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    company        date                                            content\n",
       "0  2316 楠梓電  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...\n",
       "1  2316 楠梓電  2007-01-01  台灣表面黏著科技（6278）受惠台灣近年面板蓬勃發展，去年成為最大的印刷電路板表面黏著科技廠..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news = pd.read_csv(\"data/tej_udn_all_news.csv\", encoding=\"utf-8\")\n",
    "all_news = all_news.sort_values(by=[\"date\",\"company\"]).reset_index(drop=True)\n",
    "print(all_news.shape)\n",
    "all_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>13302</td>\n",
       "      <td>13302</td>\n",
       "      <td>13302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>240</td>\n",
       "      <td>3038</td>\n",
       "      <td>12746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>2330 台積電</td>\n",
       "      <td>2008-01-17</td>\n",
       "      <td>中華信用評等公司昨日確認台塑集團四家核心公司，包括台塑、南亞塑膠、台化、台塑石化，長期企業信...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>613</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         company        date  \\\n",
       "count      13302       13302   \n",
       "unique       240        3038   \n",
       "top     2330 台積電  2008-01-17   \n",
       "freq         613          20   \n",
       "\n",
       "                                                  content  \n",
       "count                                               13302  \n",
       "unique                                              12746  \n",
       "top     中華信用評等公司昨日確認台塑集團四家核心公司，包括台塑、南亞塑膠、台化、台塑石化，長期企業信...  \n",
       "freq                                                    4  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13302 entries, 0 to 13301\n",
      "Data columns (total 3 columns):\n",
      "company    13302 non-null object\n",
      "date       13302 non-null object\n",
      "content    13302 non-null object\n",
      "dtypes: object(3)\n",
      "memory usage: 311.9+ KB\n"
     ]
    }
   ],
   "source": [
    "all_news.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文章字數 > 510了話 去尾\n",
    "# 字數小於512-2 因為還有CLS SEP\n",
    "def remove_510(text):\n",
    "    if len(text) > 510:\n",
    "        text = text[:510]    # 只取前510個字\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13302/13302 [00:00<00:00, 933190.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5469 瀚宇博</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    company        date                                            content\n",
       "0  2316 楠梓電  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...\n",
       "1  5469 瀚宇博  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news[\"content\"] = all_news[\"content\"].progress_apply(remove_510)\n",
    "print(all_news.shape)\n",
    "all_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pre-trained model tokenizer, to convert our text into tokens that correspond to BERT’s vocabulary.\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "\n",
    "def input_ids_all(text):\n",
    "#    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "    tokens = tokenizer.tokenize(text)        # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    if len(input_ids) < 512:\n",
    "        n = 512 - len(input_ids)\n",
    "        input_ids = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512\n",
    "    return input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13302/13302 [00:17<00:00, 770.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "      <th>input_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "      <td>[101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5469 瀚宇博</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "      <td>[101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    company        date                                            content  \\\n",
       "0  2316 楠梓電  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...   \n",
       "1  5469 瀚宇博  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...   \n",
       "\n",
       "                                           input_ids  \n",
       "0  [101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...  \n",
       "1  [101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df['text2'] = df['text']].apply(lambda x: \" \".join(jieba.cut(x)))\n",
    "# df_news['tokens'] = df_news['text'].apply(lambda x:  tokenizer.tokenize(x) )\n",
    "all_news['input_ids'] = all_news['content'].progress_apply(input_ids_all)\n",
    "print(all_news.shape)\n",
    "all_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 512)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 101, 1313, 1170, ...,    0,    0,    0],\n",
       "       [ 101, 1313, 1170, ...,    0,    0,    0],\n",
       "       [ 101, 1313, 1170, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101, 1377, 2768, ...,    0,    0,    0],\n",
       "       [ 101, 8020, 8021, ...,    0,    0,    0],\n",
       "       [ 101, 3189, 3315, ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# 只好用for loop一個一個拿出來合併\n",
    "# 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "input_ids = []      # list\n",
    "for index, row in all_news.iterrows():  \n",
    "    np_1d = all_news.loc[index,'input_ids']    # 1d np arrary\n",
    "    input_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "input_ids = np.array(input_ids)                       # 轉成2d np\n",
    "print(input_ids.shape)\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 512)]             0         \n",
      "_________________________________________________________________\n",
      "model (Model)                (None, 5)                 104233733 \n",
      "=================================================================\n",
      "Total params: 104,233,733\n",
      "Trainable params: 104,233,733\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Recreate the exact same model, including its weights and the optimizer\n",
    "# 讀取存的model\n",
    "input_layer = Input(shape = (512,), dtype='int64')  # 預設是float 要改成input_id的int64 # tensorflow2.0 bug 修正後可能不需要這行\n",
    "load_model = tf.keras.models.load_model('model/model_bert_eland_softmax_1')(input_layer)\n",
    "model_1 = Model(inputs=input_layer, outputs=load_model) # tensorflow2.0 bug 修正後可能不需要這行\n",
    "\n",
    "# Show the model architecture\n",
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 5)\n",
      "CPU times: user 1min 32s, sys: 22.6 s, total: 1min 54s\n",
      "Wall time: 4min 43s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[9.9999940e-01, 3.0025442e-15, 5.4861158e-07, 7.8284220e-16,\n",
       "        8.5143589e-14],\n",
       "       [9.9999952e-01, 1.7350038e-15, 4.4569668e-07, 1.8591876e-15,\n",
       "        1.1685662e-13],\n",
       "       [9.9999952e-01, 2.0303369e-15, 4.5428749e-07, 4.6447030e-16,\n",
       "        2.1901956e-14],\n",
       "       ...,\n",
       "       [1.0000000e+00, 1.8669188e-32, 8.6114038e-23, 3.9657870e-38,\n",
       "        6.4282299e-29],\n",
       "       [1.0000000e+00, 2.0209136e-20, 2.5738619e-12, 3.7298192e-24,\n",
       "        5.7666145e-14],\n",
       "       [8.9137890e-22, 2.7357174e-22, 6.2450786e-17, 5.4473147e-28,\n",
       "        1.0000000e+00]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "predictions = model_1.predict(input_ids)   # 輸出的是n*5的編碼值array\n",
    "print(predictions.shape)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>3.002544e-15</td>\n",
       "      <td>5.486116e-07</td>\n",
       "      <td>7.828422e-16</td>\n",
       "      <td>8.514359e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.735004e-15</td>\n",
       "      <td>4.456967e-07</td>\n",
       "      <td>1.859188e-15</td>\n",
       "      <td>1.168566e-13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        c_0           c_1           c_2           c_3           c_4\n",
       "0  0.999999  3.002544e-15  5.486116e-07  7.828422e-16  8.514359e-14\n",
       "1  1.000000  1.735004e-15  4.456967e-07  1.859188e-15  1.168566e-13"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_predictions = pd.DataFrame(predictions, columns = [\"c_0\", \"c_1\", \"c_2\", \"c_3\", \"c_4\"])\n",
    "print(df_predictions.shape)\n",
    "df_predictions.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13302, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "      <td>[101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>3.002544e-15</td>\n",
       "      <td>5.486116e-07</td>\n",
       "      <td>7.828422e-16</td>\n",
       "      <td>8.514359e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5469 瀚宇博</td>\n",
       "      <td>2007-01-01</td>\n",
       "      <td>印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...</td>\n",
       "      <td>[101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.735004e-15</td>\n",
       "      <td>4.456967e-07</td>\n",
       "      <td>1.859188e-15</td>\n",
       "      <td>1.168566e-13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    company        date                                            content  \\\n",
       "0  2316 楠梓電  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...   \n",
       "1  5469 瀚宇博  2007-01-01  印刷電路板產業「大者恆大」趨勢愈來愈明顯，即使景氣連續兩年上升，大廠對於今年不看淡，下游手機...   \n",
       "\n",
       "                                           input_ids       c_0           c_1  \\\n",
       "0  [101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...  0.999999  3.002544e-15   \n",
       "1  [101, 1313, 1170, 7442, 6662, 3352, 4496, 3511...  1.000000  1.735004e-15   \n",
       "\n",
       "            c_2           c_3           c_4  \n",
       "0  5.486116e-07  7.828422e-16  8.514359e-14  \n",
       "1  4.456967e-07  1.859188e-15  1.168566e-13  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news = pd.concat([all_news, df_predictions], axis=1)   # axis=1左右合併 axis=0 上下合併\n",
    "print(all_news.shape)\n",
    "all_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9402.648"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_news[\"c_0\"].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 需要宣告一個240公司*10年的dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['1101 台泥', '1201 味全', '1210 大成', '1215 卜蜂', '1216 統一', '1218 泰山',\n",
       "       '1220 台榮', '1227 佳格', '1232 大統益', '1233 天仁', '1236 宏亞', '1301 台塑',\n",
       "       '1307 三芳', '1309 台達化', '1310 台苯', '1314 中石化', '1316 上曜', '1321 大洋',\n",
       "       '1323 永裕', '1326 台化', '1413 宏洲', '1414 東和', '1432 大魯閣', '1442 名軒',\n",
       "       '1447 力鵬', '1449 佳和', '1451 年興', '1452 宏益', '1453 大將', '1456 怡華',\n",
       "       '1457 宜進', '1472 三洋紡', '1475 本盟', '1477 聚陽', '1503 士電', '1506 正道',\n",
       "       '1507 永大', '1513 中興電', '1515 力山', '1516 川飛', '1521 大億', '1525 江申',\n",
       "       '1529 樂士', '1532 勤美', '1535 中宇', '1538 正峰新', '1539 巨庭', '1558 伸興',\n",
       "       '1560 中砂', '1583 程泰', '1611 中電', '1612 宏泰', '1614 三洋電', '1618 合機',\n",
       "       '1701 中化', '1704 榮化', '1707 葡萄王', '1718 中纖', '1721 三晃', '1723 中碳',\n",
       "       '1724 台硝', '1726 永記', '1732 毛寶', '1733 五鼎', '1762 中化生', '1786 科妍',\n",
       "       '1805 寶徠', '1902 台紙', '1903 士紙', '1904 正隆', '1905 華紙', '1907 永豐餘',\n",
       "       '2002 中鋼', '2008 高興昌', '2013 中鋼構', '2020 美亞', '2025 千興',\n",
       "       '2027 大成鋼', '2028 威致', '2034 允強', '2049 上銀', '2059 川湖', '2062 橋椿',\n",
       "       '2103 台橡', '2104 國際中橡', '2105 正新', '2204 中華', '2206 三陽工業',\n",
       "       '2301 光寶科', '2308 台達電', '2311 日月光', '2314 台揚', '2316 楠梓電',\n",
       "       '2323 中環', '2324 仁寶', '2325 矽品', '2330 台積電', '2331 精英', '2332 友訊',\n",
       "       '2337 旺宏', '2338 光罩', '2340 光磊', '2344 華邦電', '2348 海悅', '2353 宏碁',\n",
       "       '2358 廷鑫', '2371 大同', '2376 技嘉', '2383 台光電', '2388 威盛', '2390 云辰',\n",
       "       '2392 正崴', '2408 南亞科', '2409 友達', '2412 中華電', '2419 仲琦',\n",
       "       '2429 銘旺科', '2438 翔耀', '2439 美律', '2444 兆勁', '2448 晶電', '2450 神腦',\n",
       "       '2451 創見', '2461 光群雷', '2472 立隆電', '2474 可成', '2475 華映', '2484 希華',\n",
       "       '2485 兆赫', '2486 一詮', '2491 吉祥全', '2498 宏達電', '2504 國產', '2506 太設',\n",
       "       '2509 全坤建', '2511 太子', '2515 中工', '2527 宏璟', '2528 皇普', '2534 宏盛',\n",
       "       '2536 宏普', '2540 愛山林', '2547 日勝生', '2606 裕民', '2610 華航', '2611 志信',\n",
       "       '2612 中航', '2613 中櫃', '2616 山隆', '2633 台灣高鐵', '2636 台驊投控',\n",
       "       '2642 宅配通', '2705 六福', '2727 王品', '2739 寒舍', '2841 台開', '2905 三商',\n",
       "       '2915 潤泰全', '3004 豐達科', '3008 大立光', '3015 全漢', '3018 同開',\n",
       "       '3026 禾伸堂', '3027 盛達', '3036 文曄', '3037 欣興', '3041 揚智', '3045 台灣大',\n",
       "       '3049 和鑫', '3052 夆典', '3054 立萬利', '3058 立德', '3090 日電貿',\n",
       "       '3130 一零四', '3149 正達', '3167 大量', '3321 同泰', '3406 玉晶光', '3494 誠研',\n",
       "       '3528 安馳', '3532 台勝科', '3591 艾笛森', '3593 力銘', '3605 宏致',\n",
       "       '3673 TPK-KY', '3682 亞太電', '3701 大眾控', '3702 大聯大', '3705 永信',\n",
       "       '4119 旭富', '4141 龍燈-KY', '4414 如興', '4532 瑞智', '4746 台耀',\n",
       "       '4906 正文', '4919 新唐', '4934 太極', '4938 和碩', '4956 光鋐', '5305 敦南',\n",
       "       '5388 中磊', '5469 瀚宇博', '5521 工信', '5608 四維航', '6128 上福', '6142 友勁',\n",
       "       '6152 百一', '6165 捷泰', '6168 宏齊', '6176 瑞儀', '6184 大豐電', '6192 巨路',\n",
       "       '6196 帆宣', '6209 今國光', '6225 天瀚', '6239 力成', '6269 台郡', '6271 同欣電',\n",
       "       '6277 宏正', '6281 全國電', '6285 啟碁', '6443 元晶', '6464 台數科',\n",
       "       '6505 台塑化', '8011 台通', '8021 尖點', '8039 台虹', '8150 南茂', '8271 宇瞻',\n",
       "       '8464 億豐', '8473 山林水', '9906 欣巴巴', '9911 櫻花', '9917 中保',\n",
       "       '9919 康那香', '9921 巨大', '9933 中鼎', '9934 成霖', '9943 好樂迪',\n",
       "       '9946 三發地產'], dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_list = all_news.sort_values(by=[\"company\",\"date\"])[\"company\"].unique()  # 把欄位unique的值都挑出來變成numpy\n",
    "print(len(company_list))\n",
    "company_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 240/240 [00:00<00:00, 59098.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([['1101 台泥', '2008'],\n",
       "       ['1101 台泥', '2009'],\n",
       "       ['1101 台泥', '2010'],\n",
       "       ...,\n",
       "       ['9946 三發地產', '2015'],\n",
       "       ['9946 三發地產', '2016'],\n",
       "       ['9946 三發地產', '2017']], dtype='<U11')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = []\n",
    "for company_name in tqdm(company_list):\n",
    "    for year in range(2008,2018):\n",
    "        company_name = str(company_name)\n",
    "        year = int(year)\n",
    "        temp_np = np.array([company_name , year])    # np(公司名,年分)\n",
    "        temp.append(temp_np)\n",
    "        \n",
    "temp = np.array(temp)\n",
    "print(len(temp))\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2400, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2395</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2398</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        company  date\n",
       "0       1101 台泥  2008\n",
       "1       1101 台泥  2009\n",
       "2       1101 台泥  2010\n",
       "3       1101 台泥  2011\n",
       "4       1101 台泥  2012\n",
       "...         ...   ...\n",
       "2395  9946 三發地產  2013\n",
       "2396  9946 三發地產  2014\n",
       "2397  9946 三發地產  2015\n",
       "2398  9946 三發地產  2016\n",
       "2399  9946 三發地產  2017\n",
       "\n",
       "[2400 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company_year_prob_sum = pd.DataFrame(temp, columns = ['company' , 'date'])  # numpy to pandas 並宣告欄位名\n",
    "print(df_company_year_prob_sum.shape)\n",
    "df_company_year_prob_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2400, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company  date  c_0  c_1  c_2  c_3  c_4\n",
       "0  1101 台泥  2008  0.0  0.0  0.0  0.0  0.0\n",
       "1  1101 台泥  2009  0.0  0.0  0.0  0.0  0.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company_year_prob_sum[\"date\"] = df_company_year_prob_sum[\"date\"].astype(int)\n",
    "df_company_year_prob_sum[\"company\"] = df_company_year_prob_sum[\"company\"].astype(str)\n",
    "df_company_year_prob_sum[\"c_0\"] = 0.       # 建class_0 ~ class_4的欄位\n",
    "df_company_year_prob_sum[\"c_1\"] = 0.\n",
    "df_company_year_prob_sum[\"c_2\"] = 0.\n",
    "df_company_year_prob_sum[\"c_3\"] = 0.\n",
    "df_company_year_prob_sum[\"c_4\"] = 0.\n",
    "print(df_company_year_prob_sum.shape)\n",
    "df_company_year_prob_sum.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>921</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      company  date  c_0  c_1  c_2  c_3  c_4\n",
       "921  2316 楠梓電  2009  0.0  0.0  0.0  0.0  0.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pandas 取出 a column contains \"A\" & b column contains \"B\" \n",
    "# df_company_year_prob_sum[(df_company_year_prob_sum[\"company\"].str.contains(\"楠梓電\")) & (df_company_year_prob_sum[\"date\"].astype(str).str.contains(\"2009\") )]\n",
    "\n",
    "df_company_year_prob_sum.loc[(df_company_year_prob_sum[\"company\"].str.contains(\"楠梓電\")) \n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains(\"2009\") )]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把每公司 每年 的 class_0相加\n",
    "### 再丟到df_topic_5_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1156</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008-01-11</td>\n",
       "      <td>華通（2313）、  （  ）、燿華（2367）及欣興（3037）等四大手機印刷電路板（PC...</td>\n",
       "      <td>[101, 5836, 6858, 8020, 10602, 8152, 8021, 510...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>5.157054e-30</td>\n",
       "      <td>5.659684e-22</td>\n",
       "      <td>3.028994e-30</td>\n",
       "      <td>1.360346e-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1171</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008-01-15</td>\n",
       "      <td>宇環（3276）近年積極籌資，去年下半年，以每股溢價12元募資1.98億元後，昨（14）日董...</td>\n",
       "      <td>[101, 2126, 4472, 8020, 12094, 8158, 8021, 681...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.418315e-32</td>\n",
       "      <td>3.552168e-24</td>\n",
       "      <td>7.187509e-35</td>\n",
       "      <td>2.747460e-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1180</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008-01-16</td>\n",
       "      <td>台灣的光電板、手機板大廠楠梓電 (2316)展開新年度人事新布局，由楠梓電集 團執行長陳梅芳...</td>\n",
       "      <td>[101, 1378, 4124, 4638, 1045, 7442, 3352, 510,...</td>\n",
       "      <td>3.471974e-17</td>\n",
       "      <td>1.122257e-14</td>\n",
       "      <td>3.181559e-08</td>\n",
       "      <td>2.757208e-20</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1187</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008-01-16</td>\n",
       "      <td>欣興（3037）昨（15）日公布去年兩岸合併營收472.81億元，穩坐台灣印刷電路板（PCB...</td>\n",
       "      <td>[101, 3615, 5646, 8020, 11068, 8161, 8021, 321...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.507751e-27</td>\n",
       "      <td>1.080081e-19</td>\n",
       "      <td>1.970381e-29</td>\n",
       "      <td>6.713191e-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1195</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008-01-17</td>\n",
       "      <td>藍營立委選舉大勝帶來台股兩天慶祝行情，但隨昨（16）日重挫，使近來一直「弱不禁風」的上市櫃印...</td>\n",
       "      <td>[101, 5965, 4245, 4989, 1999, 6908, 5647, 1920...</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.549369e-29</td>\n",
       "      <td>4.016741e-18</td>\n",
       "      <td>6.008773e-30</td>\n",
       "      <td>3.861093e-26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       company        date                                            content  \\\n",
       "1156  2316 楠梓電  2008-01-11  華通（2313）、  （  ）、燿華（2367）及欣興（3037）等四大手機印刷電路板（PC...   \n",
       "1171  2316 楠梓電  2008-01-15  宇環（3276）近年積極籌資，去年下半年，以每股溢價12元募資1.98億元後，昨（14）日董...   \n",
       "1180  2316 楠梓電  2008-01-16  台灣的光電板、手機板大廠楠梓電 (2316)展開新年度人事新布局，由楠梓電集 團執行長陳梅芳...   \n",
       "1187  2316 楠梓電  2008-01-16  欣興（3037）昨（15）日公布去年兩岸合併營收472.81億元，穩坐台灣印刷電路板（PCB...   \n",
       "1195  2316 楠梓電  2008-01-17  藍營立委選舉大勝帶來台股兩天慶祝行情，但隨昨（16）日重挫，使近來一直「弱不禁風」的上市櫃印...   \n",
       "\n",
       "                                              input_ids           c_0  \\\n",
       "1156  [101, 5836, 6858, 8020, 10602, 8152, 8021, 510...  1.000000e+00   \n",
       "1171  [101, 2126, 4472, 8020, 12094, 8158, 8021, 681...  1.000000e+00   \n",
       "1180  [101, 1378, 4124, 4638, 1045, 7442, 3352, 510,...  3.471974e-17   \n",
       "1187  [101, 3615, 5646, 8020, 11068, 8161, 8021, 321...  1.000000e+00   \n",
       "1195  [101, 5965, 4245, 4989, 1999, 6908, 5647, 1920...  1.000000e+00   \n",
       "\n",
       "               c_1           c_2           c_3           c_4  \n",
       "1156  5.157054e-30  5.659684e-22  3.028994e-30  1.360346e-30  \n",
       "1171  3.418315e-32  3.552168e-24  7.187509e-35  2.747460e-31  \n",
       "1180  1.122257e-14  3.181559e-08  2.757208e-20  1.000000e+00  \n",
       "1187  1.507751e-27  1.080081e-19  1.970381e-29  6.713191e-25  \n",
       "1195  1.549369e-29  4.016741e-18  6.008773e-30  3.861093e-26  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pandas all_news 取出 company column contains \"楠梓電\" & date column contains \"2008\" \n",
    "# df_temp = all_news[(all_news[\"company\"].str.contains(\"楠梓電\")) & (all_news[\"date\"].astype(str).str.contains(\"2008\") )]\n",
    "\n",
    "df_temp = all_news.loc[(all_news[\"company\"].str.contains(\"楠梓電\"))\n",
    "                    & (all_news[\"date\"].astype(str).str.contains(\"2008\"))]\n",
    "print(df_temp.shape)\n",
    "df_temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28.857868"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 將 c_0欄位 加總sum()\n",
    "df_temp[\"c_0\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008</td>\n",
       "      <td>28.857868</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2011</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2395</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2015</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2398</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2016</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2017</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       company  date        c_0  c_1  c_2  c_3  c_4\n",
       "0     2316 楠梓電  2008  28.857868  0.0  0.0  0.0  0.0\n",
       "1     2316 楠梓電  2009   0.000000  0.0  0.0  0.0  0.0\n",
       "2     2316 楠梓電  2010   0.000000  0.0  0.0  0.0  0.0\n",
       "3     2316 楠梓電  2011   0.000000  0.0  0.0  0.0  0.0\n",
       "4     2316 楠梓電  2012   0.000000  0.0  0.0  0.0  0.0\n",
       "...        ...   ...        ...  ...  ...  ...  ...\n",
       "2395   1453 大將  2013   0.000000  0.0  0.0  0.0  0.0\n",
       "2396   1453 大將  2014   0.000000  0.0  0.0  0.0  0.0\n",
       "2397   1453 大將  2015   0.000000  0.0  0.0  0.0  0.0\n",
       "2398   1453 大將  2016   0.000000  0.0  0.0  0.0  0.0\n",
       "2399   1453 大將  2017   0.000000  0.0  0.0  0.0  0.0\n",
       "\n",
       "[2400 rows x 7 columns]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_company_year_prob_sum[(df_company_year_prob_sum[\"company\"].str.contains(\"楠梓電\") ) & (df_company_year_prob_sum[\"date\"] == 2008) ][\"c_0\"] = df_temp[\"c_0\"].sum()\n",
    "# 填入 company欄位 contains 楠梓電 date欄位 contains 2008 的 c_0欄位\n",
    "# 沒df.loc 填入表格會失敗\n",
    "\n",
    "df_company_year_prob_sum.loc[(df_company_year_prob_sum[\"company\"].str.contains(\"楠梓電\") )\n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains( str(2008) )), \"c_0\"] = df_temp[\"c_0\"].sum()\n",
    "df_company_year_prob_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 240/240 [01:02<00:00,  3.84it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2008</td>\n",
       "      <td>28.857868</td>\n",
       "      <td>1.079127e-05</td>\n",
       "      <td>3.993868e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.148253e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2011</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2316 楠梓電</td>\n",
       "      <td>2012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2395</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2015</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2398</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2016</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>1453 大將</td>\n",
       "      <td>2017</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.656437e-35</td>\n",
       "      <td>1.880646e-24</td>\n",
       "      <td>6.978886e-35</td>\n",
       "      <td>2.031281e-34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       company  date        c_0           c_1           c_2           c_3  \\\n",
       "0     2316 楠梓電  2008  28.857868  1.079127e-05  3.993868e+00  3.000000e+00   \n",
       "1     2316 楠梓電  2009   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2     2316 楠梓電  2010   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "3     2316 楠梓電  2011   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "4     2316 楠梓電  2012   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "...        ...   ...        ...           ...           ...           ...   \n",
       "2395   1453 大將  2013   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2396   1453 大將  2014   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2397   1453 大將  2015   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2398   1453 大將  2016   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2399   1453 大將  2017   2.000000  6.656437e-35  1.880646e-24  6.978886e-35   \n",
       "\n",
       "               c_4  \n",
       "0     1.148253e+00  \n",
       "1     0.000000e+00  \n",
       "2     0.000000e+00  \n",
       "3     0.000000e+00  \n",
       "4     0.000000e+00  \n",
       "...            ...  \n",
       "2395  0.000000e+00  \n",
       "2396  0.000000e+00  \n",
       "2397  0.000000e+00  \n",
       "2398  0.000000e+00  \n",
       "2399  2.031281e-34  \n",
       "\n",
       "[2400 rows x 7 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 上面步驟寫成for loop\n",
    "# for 240公司名list:\n",
    "#     for 2008-2017年:\n",
    "\n",
    "for company_name in tqdm(company_list):\n",
    "    for year in range(2008,2018):\n",
    "        company_name = str(company_name)\n",
    "        \n",
    "        # pandas all_news 取出 company欄位 contains 公司名 & date欄位 contains 年分 \n",
    "        df_temp = all_news.loc[(all_news[\"company\"].str.contains(company_name))\n",
    "                    & (all_news[\"date\"].astype(str).str.contains( str(year) ))]\n",
    "        # 將 c_0 c_1 c_2 c_3 c_4 欄位 加總sum()\n",
    "        # df_temp[\"c_0\"].sum(), df_temp[\"c_1\"].sum(), df_temp[\"c_2\"].sum() ,df_temp[\"c_3\"].sum() ,df_temp[\"c_4\"].sum()\n",
    "\n",
    "        # 將 c_0 ~ c_4 欄位 加總sum()\n",
    "        # 按照 公司名 年分 填入 df_company_year_prob_sum的 c_0 ~ c_4 欄位表格\n",
    "        df_company_year_prob_sum.loc[(df_company_year_prob_sum[\"company\"].str.contains(company_name) )\n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains( str(year) )), \"c_0\"] = df_temp[\"c_0\"].sum()\n",
    "        df_company_year_prob_sum.loc[(df_topic_5_prob[\"company\"].str.contains(company_name) )\n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains( str(year) )), \"c_1\"] = df_temp[\"c_1\"].sum()\n",
    "        df_company_year_prob_sum.loc[(df_company_year_prob_sum[\"company\"].str.contains(company_name) )\n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains( str(year) )), \"c_2\"] = df_temp[\"c_2\"].sum()\n",
    "        df_company_year_prob_sum.loc[(df_company_year_prob_sum[\"company\"].str.contains(company_name) )\n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains( str(year) )), \"c_3\"] = df_temp[\"c_3\"].sum()\n",
    "        df_company_year_prob_sum.loc[(df_company_year_prob_sum[\"company\"].str.contains(company_name) )\n",
    "                    & (df_company_year_prob_sum[\"date\"].astype(str).str.contains( str(year) )), \"c_4\"] = df_temp[\"c_4\"].sum()\n",
    "\n",
    "df_company_year_prob_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_company_year_prob_sum.to_csv(\"data/company_year_prob_sum.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2400, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2011</td>\n",
       "      <td>42.935883</td>\n",
       "      <td>6.660758e-02</td>\n",
       "      <td>4.996792e+00</td>\n",
       "      <td>1.579325e-10</td>\n",
       "      <td>1.200072e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2012</td>\n",
       "      <td>34.983147</td>\n",
       "      <td>1.997005e+00</td>\n",
       "      <td>3.350949e+00</td>\n",
       "      <td>7.349652e-12</td>\n",
       "      <td>3.668897e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2395</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2015</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2398</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2016</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>1.470138e-13</td>\n",
       "      <td>3.596562e-07</td>\n",
       "      <td>4.796958e-23</td>\n",
       "      <td>1.999998e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.433146e-20</td>\n",
       "      <td>1.467837e-13</td>\n",
       "      <td>2.940595e-25</td>\n",
       "      <td>2.578725e-17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        company  date        c_0           c_1           c_2           c_3  \\\n",
       "0       1101 台泥  2008   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "1       1101 台泥  2009   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2       1101 台泥  2010   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "3       1101 台泥  2011  42.935883  6.660758e-02  4.996792e+00  1.579325e-10   \n",
       "4       1101 台泥  2012  34.983147  1.997005e+00  3.350949e+00  7.349652e-12   \n",
       "...         ...   ...        ...           ...           ...           ...   \n",
       "2395  9946 三發地產  2013   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2396  9946 三發地產  2014   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2397  9946 三發地產  2015   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2398  9946 三發地產  2016   0.000002  1.470138e-13  3.596562e-07  4.796958e-23   \n",
       "2399  9946 三發地產  2017   1.000000  1.433146e-20  1.467837e-13  2.940595e-25   \n",
       "\n",
       "               c_4  \n",
       "0     0.000000e+00  \n",
       "1     0.000000e+00  \n",
       "2     0.000000e+00  \n",
       "3     1.200072e+01  \n",
       "4     3.668897e+00  \n",
       "...            ...  \n",
       "2395  0.000000e+00  \n",
       "2396  0.000000e+00  \n",
       "2397  0.000000e+00  \n",
       "2398  1.999998e+00  \n",
       "2399  2.578725e-17  \n",
       "\n",
       "[2400 rows x 7 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company_year_prob_sum = pd.read_csv(\"data/company_year_prob_sum.csv\", encoding=\"utf-8\")\n",
    "df_company_year_prob_sum[\"c_0\"] = df_company_year_prob_sum[\"c_0\"].astype(float)  # 讀取後將欄位弄成float\n",
    "df_company_year_prob_sum[\"c_1\"] = df_company_year_prob_sum[\"c_1\"].astype(float)\n",
    "df_company_year_prob_sum[\"c_2\"] = df_company_year_prob_sum[\"c_2\"].astype(float)\n",
    "df_company_year_prob_sum[\"c_3\"] = df_company_year_prob_sum[\"c_3\"].astype(float)\n",
    "df_company_year_prob_sum[\"c_4\"] = df_company_year_prob_sum[\"c_4\"].astype(float)\n",
    "print(df_company_year_prob_sum.shape)\n",
    "df_company_year_prob_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1201 味全</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1210 大成</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1215 卜蜂</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1216 統一</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2359</th>\n",
       "      <td>9921 巨大</td>\n",
       "      <td>2017</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2369</th>\n",
       "      <td>9933 中鼎</td>\n",
       "      <td>2017</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2379</th>\n",
       "      <td>9934 成霖</td>\n",
       "      <td>2017</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2389</th>\n",
       "      <td>9943 好樂迪</td>\n",
       "      <td>2017</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.433146e-20</td>\n",
       "      <td>1.467837e-13</td>\n",
       "      <td>2.940595e-25</td>\n",
       "      <td>2.578725e-17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        company  date  c_0           c_1           c_2           c_3  \\\n",
       "0       1101 台泥  2008  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "10      1201 味全  2008  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "20      1210 大成  2008  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "30      1215 卜蜂  2008  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "40      1216 統一  2008  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "...         ...   ...  ...           ...           ...           ...   \n",
       "2359    9921 巨大  2017  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2369    9933 中鼎  2017  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2379    9934 成霖  2017  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2389   9943 好樂迪  2017  0.0  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2399  9946 三發地產  2017  1.0  1.433146e-20  1.467837e-13  2.940595e-25   \n",
       "\n",
       "               c_4  \n",
       "0     0.000000e+00  \n",
       "10    0.000000e+00  \n",
       "20    0.000000e+00  \n",
       "30    0.000000e+00  \n",
       "40    0.000000e+00  \n",
       "...            ...  \n",
       "2359  0.000000e+00  \n",
       "2369  0.000000e+00  \n",
       "2379  0.000000e+00  \n",
       "2389  0.000000e+00  \n",
       "2399  2.578725e-17  \n",
       "\n",
       "[2400 rows x 7 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company_year_prob_sum.sort_values(by=[\"date\",\"company\"])  # 先日期排序 再公司名排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>date</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>c_2</th>\n",
       "      <th>c_3</th>\n",
       "      <th>c_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2008</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2011</td>\n",
       "      <td>42.935883</td>\n",
       "      <td>6.660758e-02</td>\n",
       "      <td>4.996792e+00</td>\n",
       "      <td>1.579325e-10</td>\n",
       "      <td>1.200072e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2012</td>\n",
       "      <td>34.983147</td>\n",
       "      <td>1.997005e+00</td>\n",
       "      <td>3.350949e+00</td>\n",
       "      <td>7.349652e-12</td>\n",
       "      <td>3.668897e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2395</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2015</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2398</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2016</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>1.470138e-13</td>\n",
       "      <td>3.596562e-07</td>\n",
       "      <td>4.796958e-23</td>\n",
       "      <td>1.999998e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>9946 三發地產</td>\n",
       "      <td>2017</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.433146e-20</td>\n",
       "      <td>1.467837e-13</td>\n",
       "      <td>2.940595e-25</td>\n",
       "      <td>2.578725e-17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        company  date        c_0           c_1           c_2           c_3  \\\n",
       "0       1101 台泥  2008   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "1       1101 台泥  2009   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2       1101 台泥  2010   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "3       1101 台泥  2011  42.935883  6.660758e-02  4.996792e+00  1.579325e-10   \n",
       "4       1101 台泥  2012  34.983147  1.997005e+00  3.350949e+00  7.349652e-12   \n",
       "...         ...   ...        ...           ...           ...           ...   \n",
       "2395  9946 三發地產  2013   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2396  9946 三發地產  2014   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2397  9946 三發地產  2015   0.000000  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "2398  9946 三發地產  2016   0.000002  1.470138e-13  3.596562e-07  4.796958e-23   \n",
       "2399  9946 三發地產  2017   1.000000  1.433146e-20  1.467837e-13  2.940595e-25   \n",
       "\n",
       "               c_4  \n",
       "0     0.000000e+00  \n",
       "1     0.000000e+00  \n",
       "2     0.000000e+00  \n",
       "3     1.200072e+01  \n",
       "4     3.668897e+00  \n",
       "...            ...  \n",
       "2395  0.000000e+00  \n",
       "2396  0.000000e+00  \n",
       "2397  0.000000e+00  \n",
       "2398  1.999998e+00  \n",
       "2399  2.578725e-17  \n",
       "\n",
       "[2400 rows x 7 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company_year_prob_sum.sort_values(by=[\"company\",\"date\"])  # 先公司名排序 再日期排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['1101 台泥', '1201 味全', '1210 大成', '1215 卜蜂', '1216 統一', '1218 泰山',\n",
       "       '1220 台榮', '1227 佳格', '1232 大統益', '1233 天仁', '1236 宏亞', '1301 台塑',\n",
       "       '1307 三芳', '1309 台達化', '1310 台苯', '1314 中石化', '1316 上曜', '1321 大洋',\n",
       "       '1323 永裕', '1326 台化', '1413 宏洲', '1414 東和', '1432 大魯閣', '1442 名軒',\n",
       "       '1447 力鵬', '1449 佳和', '1451 年興', '1452 宏益', '1453 大將', '1456 怡華',\n",
       "       '1457 宜進', '1472 三洋紡', '1475 本盟', '1477 聚陽', '1503 士電', '1506 正道',\n",
       "       '1507 永大', '1513 中興電', '1515 力山', '1516 川飛', '1521 大億', '1525 江申',\n",
       "       '1529 樂士', '1532 勤美', '1535 中宇', '1538 正峰新', '1539 巨庭', '1558 伸興',\n",
       "       '1560 中砂', '1583 程泰', '1611 中電', '1612 宏泰', '1614 三洋電', '1618 合機',\n",
       "       '1701 中化', '1704 榮化', '1707 葡萄王', '1718 中纖', '1721 三晃', '1723 中碳',\n",
       "       '1724 台硝', '1726 永記', '1732 毛寶', '1733 五鼎', '1762 中化生', '1786 科妍',\n",
       "       '1805 寶徠', '1902 台紙', '1903 士紙', '1904 正隆', '1905 華紙', '1907 永豐餘',\n",
       "       '2002 中鋼', '2008 高興昌', '2013 中鋼構', '2020 美亞', '2025 千興',\n",
       "       '2027 大成鋼', '2028 威致', '2034 允強', '2049 上銀', '2059 川湖', '2062 橋椿',\n",
       "       '2103 台橡', '2104 國際中橡', '2105 正新', '2204 中華', '2206 三陽工業',\n",
       "       '2301 光寶科', '2308 台達電', '2311 日月光', '2314 台揚', '2316 楠梓電',\n",
       "       '2323 中環', '2324 仁寶', '2325 矽品', '2330 台積電', '2331 精英', '2332 友訊',\n",
       "       '2337 旺宏', '2338 光罩', '2340 光磊', '2344 華邦電', '2348 海悅', '2353 宏碁',\n",
       "       '2358 廷鑫', '2371 大同', '2376 技嘉', '2383 台光電', '2388 威盛', '2390 云辰',\n",
       "       '2392 正崴', '2408 南亞科', '2409 友達', '2412 中華電', '2419 仲琦',\n",
       "       '2429 銘旺科', '2438 翔耀', '2439 美律', '2444 兆勁', '2448 晶電', '2450 神腦',\n",
       "       '2451 創見', '2461 光群雷', '2472 立隆電', '2474 可成', '2475 華映', '2484 希華',\n",
       "       '2485 兆赫', '2486 一詮', '2491 吉祥全', '2498 宏達電', '2504 國產', '2506 太設',\n",
       "       '2509 全坤建', '2511 太子', '2515 中工', '2527 宏璟', '2528 皇普', '2534 宏盛',\n",
       "       '2536 宏普', '2540 愛山林', '2547 日勝生', '2606 裕民', '2610 華航', '2611 志信',\n",
       "       '2612 中航', '2613 中櫃', '2616 山隆', '2633 台灣高鐵', '2636 台驊投控',\n",
       "       '2642 宅配通', '2705 六福', '2727 王品', '2739 寒舍', '2841 台開', '2905 三商',\n",
       "       '2915 潤泰全', '3004 豐達科', '3008 大立光', '3015 全漢', '3018 同開',\n",
       "       '3026 禾伸堂', '3027 盛達', '3036 文曄', '3037 欣興', '3041 揚智', '3045 台灣大',\n",
       "       '3049 和鑫', '3052 夆典', '3054 立萬利', '3058 立德', '3090 日電貿',\n",
       "       '3130 一零四', '3149 正達', '3167 大量', '3321 同泰', '3406 玉晶光', '3494 誠研',\n",
       "       '3528 安馳', '3532 台勝科', '3591 艾笛森', '3593 力銘', '3605 宏致',\n",
       "       '3673 TPK-KY', '3682 亞太電', '3701 大眾控', '3702 大聯大', '3705 永信',\n",
       "       '4119 旭富', '4141 龍燈-KY', '4414 如興', '4532 瑞智', '4746 台耀',\n",
       "       '4906 正文', '4919 新唐', '4934 太極', '4938 和碩', '4956 光鋐', '5305 敦南',\n",
       "       '5388 中磊', '5469 瀚宇博', '5521 工信', '5608 四維航', '6128 上福', '6142 友勁',\n",
       "       '6152 百一', '6165 捷泰', '6168 宏齊', '6176 瑞儀', '6184 大豐電', '6192 巨路',\n",
       "       '6196 帆宣', '6209 今國光', '6225 天瀚', '6239 力成', '6269 台郡', '6271 同欣電',\n",
       "       '6277 宏正', '6281 全國電', '6285 啟碁', '6443 元晶', '6464 台數科',\n",
       "       '6505 台塑化', '8011 台通', '8021 尖點', '8039 台虹', '8150 南茂', '8271 宇瞻',\n",
       "       '8464 億豐', '8473 山林水', '9906 欣巴巴', '9911 櫻花', '9917 中保',\n",
       "       '9919 康那香', '9921 巨大', '9933 中鼎', '9934 成霖', '9943 好樂迪',\n",
       "       '9946 三發地產'], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_list = df_company_year_prob_sum[\"company\"].unique()  # 把欄位unique的值都挑出來變成numpy\n",
    "print(len(company_list))\n",
    "company_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}