2.tokenize_tf_rtx2080ti.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Nov 25 20:10:49 2019       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  GeForce RTX 208...  On   | 00000000:01:00.0  On |                  N/A |\n",
      "|  0%   41C    P8    11W / 300W |     89MiB / 10988MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|    0      1845      G   /usr/lib/xorg/Xorg                            21MiB |\n",
      "|    0      1937      G   /usr/bin/gnome-shell                          66MiB |\n",
      "+-----------------------------------------------------------------------------+\n",
      "\n",
      "Architecture:        x86_64\n",
      "CPU op-mode(s):      32-bit, 64-bit\n",
      "Byte Order:          Little Endian\n",
      "CPU(s):              12\n",
      "On-line CPU(s) list: 0-11\n",
      "Thread(s) per core:  2\n",
      "Core(s) per socket:  6\n",
      "Socket(s):           1\n",
      "NUMA node(s):        1\n",
      "Vendor ID:           GenuineIntel\n",
      "CPU family:          6\n",
      "Model:               158\n",
      "Model name:          Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\n",
      "Stepping:            10\n",
      "CPU MHz:             3381.425\n",
      "CPU max MHz:         4600.0000\n",
      "CPU min MHz:         800.0000\n",
      "BogoMIPS:            6384.00\n",
      "Virtualization:      VT-x\n",
      "L1d cache:           32K\n",
      "L1i cache:           32K\n",
      "L2 cache:            256K\n",
      "L3 cache:            12288K\n",
      "NUMA node0 CPU(s):   0-11\n",
      "Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d\n",
      "\n",
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:            47G        1.1G         42G        1.9M        3.6G         45G\n",
      "Swap:          2.0G          0B        2.0G\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# # linux系統指令 可省略 win可能跑不了\n",
    "# ! nvidia-smi\n",
    "# ! lscpu\n",
    "# ! free -h\n",
    "try:\n",
    "    import os\n",
    "    f = os.popen('nvidia-smi')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "    f = os.popen('lscpu')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "    f = os.popen('free -h')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, BertModel, TFBertForSequenceClassification, AdamW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>sentiments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>-0.996732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>0.999998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...</td>\n",
       "      <td>0.793344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>大哥大，透過4G漫遊方式共網爭議，戰火延燒，威寶電信（台灣之星）及中華電信、遠傳電信等業者，...</td>\n",
       "      <td>0.979061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...</td>\n",
       "      <td>0.793344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16713</th>\n",
       "      <td>16714</td>\n",
       "      <td>3</td>\n",
       "      <td>上市公司玉晶光（3406）今天傳出裁員，對此，公司發言人趙志強表示，並非裁員而 上市公司玉晶...</td>\n",
       "      <td>0.978056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16714</th>\n",
       "      <td>16715</td>\n",
       "      <td>3</td>\n",
       "      <td>繼台企銀（2834）工會因合併問題而罷工，創金融史首例後，大眾銀行（2847）工 繼台企銀（...</td>\n",
       "      <td>-0.998512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16715</th>\n",
       "      <td>16716</td>\n",
       "      <td>3</td>\n",
       "      <td>6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年...</td>\n",
       "      <td>0.999983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16716</th>\n",
       "      <td>16717</td>\n",
       "      <td>3</td>\n",
       "      <td>智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出...</td>\n",
       "      <td>0.999967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16717</th>\n",
       "      <td>16718</td>\n",
       "      <td>3</td>\n",
       "      <td>時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從...</td>\n",
       "      <td>0.999988</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16718 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  label                                               text  \\\n",
       "0          1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   \n",
       "1          2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...   \n",
       "2          3      1  測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...   \n",
       "3          4      1  大哥大，透過4G漫遊方式共網爭議，戰火延燒，威寶電信（台灣之星）及中華電信、遠傳電信等業者，...   \n",
       "4          5      1  測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...   \n",
       "...      ...    ...                                                ...   \n",
       "16713  16714      3  上市公司玉晶光（3406）今天傳出裁員，對此，公司發言人趙志強表示，並非裁員而 上市公司玉晶...   \n",
       "16714  16715      3  繼台企銀（2834）工會因合併問題而罷工，創金融史首例後，大眾銀行（2847）工 繼台企銀（...   \n",
       "16715  16716      3  6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年...   \n",
       "16716  16717      3  智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出...   \n",
       "16717  16718      3  時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從...   \n",
       "\n",
       "       sentiments  \n",
       "0       -0.996732  \n",
       "1        0.999998  \n",
       "2        0.793344  \n",
       "3        0.979061  \n",
       "4        0.793344  \n",
       "...           ...  \n",
       "16713    0.978056  \n",
       "16714   -0.998512  \n",
       "16715    0.999983  \n",
       "16716    0.999967  \n",
       "16717    0.999988  \n",
       "\n",
       "[16718 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 讀取tsv # \\t tab做區隔\n",
    "df_news = pd.read_csv(\"https://github.com/roccqqck/news_bert/raw/master/data/2015_Company.tsv\", sep=\"\\t\", encoding=\"utf-8\")\n",
    "df_news['text'] = df_news['text'].astype(str)\n",
    "df_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    5623\n",
       "0    4877\n",
       "1    4546\n",
       "3    1316\n",
       "2     356\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文章字數 > 510了話 去尾\n",
    "# 字數小於512-2 因為還有CLS SEP\n",
    "def remove_510(text):\n",
    "    if len(text) > 510:\n",
    "        text = text[:510]    # 只取前510個字\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>sentiments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>-0.996732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>0.999998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text  sentiments\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   -0.996732\n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...    0.999998"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news[\"text\"] = df_news[\"text\"].apply(remove_510)\n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pre-trained model tokenizer, to convert our text into tokens that correspond to BERT’s vocabulary.\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![avatar](https://github.com/roccqqck/news_bert/raw/master/bert_input_encoding.jpg)\n",
    "\n",
    "https://github.com/roccqqck/news_bert/raw/master/bert_input_encoding.jpg\n",
    "\n",
    "bert input features 有3個\n",
    "\n",
    "input_ids: 代表識別每個 token 的索引值，用 tokenizer 轉換即可\n",
    "\n",
    "token_type_ids: 用來識別句子界限。第一句為 0，第二句則為 1。另外注意句子間的 [SEP] 為 0     (optional) 輸入有1句非必要 輸入有2句則必要\n",
    "\n",
    "attention_mask: 用來界定自注意力機制範圍。1 讓 BERT 關注該位置，0 則代表是 padding 不需關注\n",
    "(optional)\n",
    "\n",
    "https://huggingface.co/transformers/model_doc/bert.html#tfbertforsequenceclassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['很', '好', '看', '的', '動', '作', '片', '，', '不', '會', '浪', '費', '錢',\n",
       "       '跟', '時', '間', '。', '很', '久', '沒', '有', '這', '樣', '的', '探', '險',\n",
       "       '片', '。', '可', '說', '是', '女', '版', '的', '印', '第', '安', '那', '瓊',\n",
       "       '。'], dtype='<U1')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenize input\n",
    "text = \"很好看的動作片，不會浪費錢跟時間。很久沒有這樣的探險片。可說是女版的印第安那瓊。\"\n",
    "tokens = tokenizer.tokenize(text)      # 每個字切詞成一個list\n",
    "print(type(tokens))                 # list\n",
    "np.array(tokens)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "40\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298, 3857,\n",
       "       6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300, 6857,\n",
       "       3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957, 4276,\n",
       "       4638, 1313, 5018, 2128, 6929, 4475,  511])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.convert_tokens_to_ids(tokens)   # 每個字轉成id\n",
    "print(type(input_ids))                         # list\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids) # token_type_ids 必須input還沒加CLS SEP\n",
    "print(type(token_type_ids))                                # list\n",
    "print(len(token_type_ids)) \n",
    "np.array(token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "print(type(input_ids))\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = 512 - len(input_ids)\n",
    "input_ids2 = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "# array右邊append n 個 0  補長度到512\n",
    "print(len(input_ids2))\n",
    "input_ids2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['很', '好', '看', '的', '動', '作', '片'], dtype='<U1')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input如果是兩個句子\n",
    "\n",
    "text = \"很好看的動作片\"\n",
    "tokens = tokenizer.tokenize(text)      # 每個字切詞成一個list\n",
    "print(type(tokens))                 # list\n",
    "np.array(tokens)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['不', '會', '浪', '費', '錢', '跟', '時', '間'], dtype='<U1')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text2 = \"不會浪費錢跟時間\"\n",
    "tokens2 = tokenizer.tokenize(text2)      # 每個字切詞成一個list\n",
    "print(type(tokens2))                 # list\n",
    "np.array(tokens2)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2523, 1962, 4692, 4638, 1240,  868, 4275])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.convert_tokens_to_ids(tokens)   # 每個字轉成id\n",
    "print(type(input_ids))                         # list\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 679, 3298, 3857, 6527, 7092, 6656, 3229, 7279])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids2 = tokenizer.convert_tokens_to_ids(tokens2)   # 每個字轉成id\n",
    "print(type(input_ids2))                         # list\n",
    "print(len(input_ids2))\n",
    "np.array(input_ids2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids, input_ids2) # token_type_ids 必須input還沒加CLS SEP\n",
    "print(type(token_type_ids))                                # list\n",
    "print(len(token_type_ids)) \n",
    "np.array(token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275,  102,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  102])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids3 = tokenizer.build_inputs_with_special_tokens(input_ids, input_ids2)    # 句子前後加上 CLS SEP 的 id\n",
    "print(type(input_ids3))\n",
    "print(len(input_ids3))\n",
    "np.array(input_ids3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "numpy.pad   補0到某長度\n",
    "\n",
    "https://docs.scipy.org/doc/numpy/reference/generated/numpy.pad.html \n",
    "\n",
    "也可以使用\n",
    "\n",
    "```from keras.preprocessing.sequence import pad_sequences```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def input_ids_all(text):\n",
    "#    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "    tokens = tokenizer.tokenize(text)        # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    if len(input_ids) < 512:\n",
    "        n = 512 - len(input_ids)\n",
    "        input_ids = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512\n",
    "    return input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"很好看的動作片，不會浪費錢跟時間。很久沒有這樣的探險片。可說是女版的印第安那瓊。\"\n",
    "input_ids_all(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention_mask_all(text):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    attention_mask = np.array([1,1])\n",
    "    attention_mask = np.pad(attention_mask, (0, len(input_ids)-2 ), mode ='constant', constant_values=(1)) \n",
    "    # array右邊append 1 到跟segment一樣長\n",
    "    if len(attention_mask) < 512:\n",
    "        n = 512 - len(attention_mask)\n",
    "        attention_mask = np.pad(attention_mask, (0, n), mode ='constant', constant_values=(0))  # array右邊append n 個 0  補長度到512\n",
    "    return attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask_all(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 事實上單一句子　出來都是0　不做也沒差\n",
    "\n",
    "def token_type_ids_all(text):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids)   # token_type_ids 必須input還沒加CLS SEP\n",
    "    token_type_ids = np.array(token_type_ids)              # list 轉numpy\n",
    "    if len(token_type_ids) < 512:\n",
    "        n = 512 - len(token_type_ids)\n",
    "        token_type_ids = np.pad(token_type_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512    \n",
    "    return token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids_all(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下來要所有df_news['text']一起做\n",
    "\n",
    "用list的apply\n",
    "\n",
    "[y for y in [1,3,5,7,9]] 但這個有點難理解 不直覺"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, 5, 7, 9]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[y for y in [1,3,5,7,9] ]         # 輸出是list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 6, 10, 14, 18]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sq(x):                       # 定義func\n",
    "    return x * 2\n",
    "\n",
    "[sq(y) for y in [1,3,5,7,9]]       # 全部 經過func *2 並且輸出成一個list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，頂率還款將不影響味全6千名員工的生計。兆豐銀行表示，頂率開發聯貸屆期違約案，經聯貸銀行團決議發出催告存證信函，請',\n",
       "       '至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團在元旦假期後進行法律程序，他認為頂率土地拍賣後，應足夠償還六十五億元債務，不影響味全員工生計。頂新集團副總賈先德 力協助，但看來魏家寧可花心思賣頂率資產及台北101股權，也不願直接從海外匯錢，擺明就是不想從自己口袋裡掏錢。對於味全有再標售三重土地及頂率股權，吳當傑說：「聯貸違約，一旦債權銀行團執行假扣押，就不能自己標 向板橋地院聲請假扣押三重聯貸案土地●財政部立場．尊重銀行團決議，元旦假期後積極法催．三重土地拍賣足夠償還債權，不影響味全員工生計．若執行假扣押，頂新不能賣土地，違約仍須還錢●未來發展．執行假扣押之前，銀行 三重土地拍賣足夠償還債權，不影響味全員工生計．若執行假扣押，頂新不能賣土地，違約仍須還錢●未來發展．執行假扣押之前，銀行仍歡迎頂新還錢．現行聯貸利率約2.2％，違約後利率將調升至3％以上資料來源：採訪整理',\n",
       "       '測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為如IMEI、IMSI或定位資訊，或系統更新及加密資料等資訊，皆未有涉違反個資法的情況出現。另針對鴻海集團旗下亞太電信、',\n",
       "       ...,\n",
       "       '6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年首季挑戰損益兩平，終結連續3季的大虧局面，不過這回一週連兩位中階主管猝死，宏達電內部的人力和手機研發對王雪紅來說',\n",
       "       '智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出身體不適，疑似過勞猝死，宏達電方面聲明，員工的工作時間都很正常 智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出身體不適，疑似過勞猝死，宏達電方面聲明，員工的工作時間都很正常，不過勞團警告，科技業頻頻傳出過勞悲劇，恐怕都是 接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出身體不適，疑似過勞猝死，宏達電方面聲明，員工的工作時間都很正常，不過勞團警告，科技業頻頻傳出過勞悲劇，恐怕都是責任制惹的禍！帶領員工一起 間都很正常，不過勞團警告，科技業頻頻傳出過勞悲劇，恐怕都是責任制惹的禍！帶領員工一起跳舞，宏達電董事長王雪紅，才參加完聖誕節慶祝活動。沒想到，就有員工向周刊投訴過勞猝死。一位任職長達17年，研發單位的何姓經理 時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從來都沒有失敗過喔。」宏達電傳勞工爭議不是第一次，先前還傳出今年九月，裁員將近15%，另外五年前，南亞科也有員工猝 紅(2012.08.28)：「我們宏達',\n",
       "       '時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從來都沒有失敗過喔。」宏達電傳勞工爭議不是第一次，先前還傳出今年九月，裁員將近15%，另外五年前，南亞科也有員工猝 紅(2012.08.28)：「我們宏達電一向都是從來都沒有失敗過喔。」宏達電傳勞工爭議不是第一次，先前還傳出今年九月，裁員將近15%，另外五年前，南亞科也有員工猝死家中。職安連線執行長黃怡翎：「發病前的一 %，另外五年前，南亞科也有員工猝死家中。職安連線執行長黃怡翎：「發病前的一個月，加班時數超過92小時，或是發病前的二到六個月，加班時數超過72小時，那原則上都是過勞的認定標準。」拚業績之外怎麼，兼顧員工健'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = df_news[\"text\"].to_numpy()     # 1d numpy 的df_news[\"news\"]\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = [ input_ids_all(i) for i in text]      # 必須要 [ ] 輸出是list\n",
    "input_ids = np.array(input_ids)                    # 轉成numpy\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最後我決定用pandas的apply 比較好視覺化理解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>sentiments</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>attention_mask</th>\n",
       "      <th>token_type_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>-0.996732</td>\n",
       "      <td>[101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>[101, 5635, 676, 6640, 809, 677, 8024, 519, 17...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text  sentiments  \\\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   -0.996732   \n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...    0.999998   \n",
       "\n",
       "                                           input_ids  \\\n",
       "0  [101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...   \n",
       "1  [101, 5635, 676, 6640, 809, 677, 8024, 519, 17...   \n",
       "\n",
       "                                      attention_mask  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                      token_type_ids  \n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df['text2'] = df['text']].apply(lambda x: \" \".join(jieba.cut(x)))\n",
    "# df_news['tokens'] = df_news['text'].apply(lambda x:  tokenizer.tokenize(x) )\n",
    "df_news['input_ids'] = df_news['text'].apply(input_ids_all)\n",
    "df_news['attention_mask'] = df_news['text'].apply(attention_mask_all)\n",
    "df_news['token_type_ids'] = df_news['text'].apply(token_type_ids_all)\n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_news['input_ids'].to_numpy()   # 提出來 竟然不是2d numpy 不能這樣做"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 6512, 3124, 6956, 1041, 1146, 2203, 7028, 7065, 6121, 1757,\n",
       "       4825,  924, 1002, 3609, 4638, 2553, 6206,  868, 4158, 8024, 7521,\n",
       "       6243, 1039, 3190, 2527, 2218, 3298, 6313, 3791, 7368, 2179, 3177,\n",
       "        969, 2807, 2852, 8024, 1963,  676, 7028, 1759, 1765, 7518, 1164,\n",
       "       2864, 6546, 8024, 7515, 4372, 6917, 3621, 2200,  679, 2512, 7513,\n",
       "       1456, 1059,  127, 1283, 1399, 1519, 2339, 4638, 4495, 6243,  511,\n",
       "       1042, 6493, 7065, 6121, 6134, 4850, 8024, 7515, 4372, 7274, 4634,\n",
       "       5474, 6526, 2234, 3309, 6889, 5147, 3428, 8024, 5195, 5474, 6526,\n",
       "       7065, 6121, 1757, 3748, 6359, 4634, 1139,  998, 1440, 2100, 6349,\n",
       "        928, 1141, 8024, 6313,  102,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['input_ids'][0]))\n",
    "df_news['input_ids'][0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['token_type_ids'][0]))\n",
    "df_news['token_type_ids'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['attention_mask'][0]))\n",
    "df_news['attention_mask'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2個column numpy相加\n",
    "def add_2_column(input_ids, attention_mask):\n",
    "    add = np.array(input_ids) + np.array(attention_mask)\n",
    "    return add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>sentiments</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>attention_mask</th>\n",
       "      <th>token_type_ids</th>\n",
       "      <th>add</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>-0.996732</td>\n",
       "      <td>[101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[102, 6513, 3125, 6957, 1042, 1147, 2204, 7029...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>[101, 5635, 676, 6640, 809, 677, 8024, 519, 17...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[102, 5636, 677, 6641, 810, 678, 8025, 520, 17...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text  sentiments  \\\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   -0.996732   \n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...    0.999998   \n",
       "\n",
       "                                           input_ids  \\\n",
       "0  [101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...   \n",
       "1  [101, 5635, 676, 6640, 809, 677, 8024, 519, 17...   \n",
       "\n",
       "                                      attention_mask  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                      token_type_ids  \\\n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                                                 add  \n",
       "0  [102, 6513, 3125, 6957, 1042, 1147, 2204, 7029...  \n",
       "1  [102, 5636, 677, 6641, 810, 678, 8025, 520, 17...  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pandas apply 用2個column # 2個column numpy相加\n",
    "df_news['add'] = df_news.apply(lambda row: add_2_column(row['input_ids'], row['attention_mask']), axis=1)   \n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input是2個句子才真的需要的token_type_ids\n",
    "\n",
    "def token_type_ids_all(text, text2):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    tokens2 = tokenizer.tokenize(text2)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids2 = tokenizer.convert_tokens_to_ids(tokens2)  # 每個字轉成id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    input_ids2 = np.array(input_ids2)          # list 轉 numpy\n",
    "    token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids, input_ids2)   # token_type_ids 必須input還沒加CLS SEP\n",
    "    token_type_ids = np.array(token_type_ids)              # list 轉numpy\n",
    "    if len(token_type_ids) < 512:\n",
    "        n = 512 - len(token_type_ids)\n",
    "        token_type_ids = np.pad(token_type_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512    \n",
    "    return token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>sentiments</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>attention_mask</th>\n",
       "      <th>token_type_ids</th>\n",
       "      <th>add</th>\n",
       "      <th>text2</th>\n",
       "      <th>token_type_ids2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>-0.996732</td>\n",
       "      <td>[101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[102, 6513, 3125, 6957, 1042, 1147, 2204, 7029...</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>[101, 5635, 676, 6640, 809, 677, 8024, 519, 17...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[102, 5636, 677, 6641, 810, 678, 8025, 520, 17...</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text  sentiments  \\\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   -0.996732   \n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...    0.999998   \n",
       "\n",
       "                                           input_ids  \\\n",
       "0  [101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...   \n",
       "1  [101, 5635, 676, 6640, 809, 677, 8024, 519, 17...   \n",
       "\n",
       "                                      attention_mask  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                      token_type_ids  \\\n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                                                 add  \\\n",
       "0  [102, 6513, 3125, 6957, 1042, 1147, 2204, 7029...   \n",
       "1  [102, 5636, 677, 6641, 810, 678, 8025, 520, 17...   \n",
       "\n",
       "                                               text2  \\\n",
       "0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   \n",
       "1  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...   \n",
       "\n",
       "                                     token_type_ids2  \n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pandas apply 用2個column 做token_type_ids\n",
    "\n",
    "df_news['text2'] = df_news['text']\n",
    "df_news['token_type_ids2'] = df_news.apply(lambda row: token_type_ids_all(row['text'], row['text2']), axis=1)   \n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news['token_type_ids2'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_news = df_news.drop(columns=[\"add\", \"text2\", \"token_type_ids2\" ])     # 刪除3個column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# # 只好用for loop一個一個拿出來合併\n",
    "# # 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "\n",
    "# input_ids = np.zeros((1, 512)).astype(int) #宣吿一個都是0的1*512 numpy # np.zeros預設是float 改成int 不然bert餵不進去\n",
    "# for index, row in df_news.iterrows():  \n",
    "#     element = df_news.loc[index,'input_ids']\n",
    "#     input_ids = np.vstack((input_ids, np.array([element])))   # 2維 合併\n",
    "\n",
    "# input_ids = np.delete(input_ids, 0, 0)              # 刪掉一開始都是0的那一個宣告  \n",
    "# input_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.quora.com/Is-it-better-to-use-np-append-or-list-append"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# 只好用for loop一個一個拿出來合併\n",
    "# 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "input_ids = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'input_ids']    # 1d np arrary\n",
    "    input_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "input_ids = np.array(input_ids)                       # 轉成2d np\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.tensor(input_ids)    # numpy 轉 torch tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=2, shape=(16718, 512), dtype=int64, numpy=\n",
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.convert_to_tensor(input_ids)   # numpy 轉 tf tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'attention_mask']    # 1d np arrary\n",
    "    attention_mask.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "attention_mask = np.array(attention_mask)                       # 轉成2d np\n",
    "attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'token_type_ids']    # 1d np arrary\n",
    "    token_type_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "token_type_ids = np.array(token_type_ids)                       # 轉成2d np\n",
    "token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, ..., 3, 3, 3])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label = df_news['label'].to_numpy()\n",
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把2個input_ids, attention_mask , token_type_ids 還有label 切成training data, validation data\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Use train_test_split to split our data into train and validation sets for training\n",
    "\n",
    "# # 設定 stratify = label 把每個類別平均\n",
    "train_input_ids, validation_input_ids, train_label, validation_label = train_test_split(input_ids, label, \n",
    "                                                            random_state=2018, test_size=0.5, stratify=label )\n",
    "\n",
    "train_attention_mask, validation_attention_mask, _, _ = train_test_split(attention_mask, label,\n",
    "                                             random_state=2018, test_size=0.5, stratify=label )\n",
    "\n",
    "train_token_type_ids, validation_token_type_ids, _, _ = train_test_split(token_type_ids, label,\n",
    "                                             random_state=2018, test_size=0.5, stratify=label )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # input 可用 numpy 或 tf tensor 下面是numpy轉tf tensor\n",
    "\n",
    "# train_input_ids = tf.convert_to_tensor(train_input_ids)\n",
    "# validation_input_ids = tf.convert_to_tensor(validation_input_ids)\n",
    "# train_label = tf.convert_to_tensor(train_label)\n",
    "# validation_label = tf.convert_to_tensor(validation_label)\n",
    "# train_attention_mask = tf.convert_to_tensor(train_attention_mask)\n",
    "# validation_attention_mask = tf.convert_to_tensor(validation_attention_mask)\n",
    "# train_token_type_ids = tf.convert_to_tensor(train_token_type_ids)\n",
    "# validation_token_type_ids = tf.convert_to_tensor(validation_token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"tf_bert_for_sequence_classification\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "bert (TFBertMainLayer)       multiple                  102267648 \n",
      "_________________________________________________________________\n",
      "dropout_37 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "classifier (Dense)           multiple                  3845      \n",
      "=================================================================\n",
      "Total params: 102,271,493\n",
      "Trainable params: 102,271,493\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule \n",
    "# num_labels=5 分5類\n",
    "model = TFBertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=5)\n",
    "model.summary()\n",
    "\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8359 samples, validate on 8359 samples\n",
      "Epoch 1/4\n",
      "8359/8359 [==============================] - 754s 90ms/sample - loss: 0.3190 - accuracy: 0.8954 - val_loss: 0.1887 - val_accuracy: 0.9303\n",
      "Epoch 2/4\n",
      "8359/8359 [==============================] - 745s 89ms/sample - loss: 0.1357 - accuracy: 0.9579 - val_loss: 0.1248 - val_accuracy: 0.9569\n",
      "Epoch 3/4\n",
      "8359/8359 [==============================] - 745s 89ms/sample - loss: 0.1154 - accuracy: 0.9654 - val_loss: 0.1408 - val_accuracy: 0.9521\n",
      "Epoch 4/4\n",
      "8359/8359 [==============================] - 748s 90ms/sample - loss: 0.1054 - accuracy: 0.9677 - val_loss: 0.1417 - val_accuracy: 0.9617\n",
      "CPU times: user 55min 11s, sys: 6min 33s, total: 1h 1min 44s\n",
      "Wall time: 49min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train and evaluate using tf.keras.Model.fit()  # batch size 8就會error 是GPU記憶體爆掉\n",
    "model_fit = model.fit(train_input_ids, train_label, \n",
    "                      batch_size=4, epochs=4, \n",
    "                    validation_data=(validation_input_ids, validation_label)\n",
    "#                    steps_per_epoch=115,\n",
    "#                    validation_steps=7)\n",
    "                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://huggingface.co/transformers/model_doc/bert.html#tfbertforsequenceclassification\n",
    "\n",
    "attention_mask 或 token_type_ids 不一定要放輸入 可選擇\n",
    "\n",
    "要放了話要加```[ ]```\n",
    "\n",
    "```model.fit([train_input_ids, train_attention_mask, train_token_type_ids], train_label)```\n",
    "\n",
    "就是```model.fit(X_train, Y_train)```\n",
    "\n",
    "```[train_input_ids, train_attention_mask, train_token_type_ids]``` 就是 ```X_train```\n",
    "\n",
    "```train_label``` 就是 ```Y_train```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_fit = model.fit([train_input_ids, train_attention_mask, train_token_type_ids], train_label, \n",
    "#                        batch_size=4, epochs=1, \n",
    "#                     validation_data=([validation_input_ids, validation_attention_mask, validation_token_type_ids], validation_label)\n",
    "#                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 如果train到一半 想要重新train 在jupyter裡面interrupt kernel\n",
    "# # 這時候 model還是存在在記憶體裡面 只是train到一半 要重新train要釋放model的記憶體\n",
    "# del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.evaluate(validation_input_ids, validation_label, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://blog.csdn.net/zds13257177985/article/details/80638384\n",
    "\n",
    "```predictions = model.predict（test）```預測的是數值，而且輸出的是n*5的編碼值array\n",
    "\n",
    "要經過```predictions = np.argmax(predictions, axis=1)```才是類別"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.8 s, sys: 8.8 s, total: 1min 7s\n",
      "Wall time: 2min 7s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-1.6275736,  1.0288228, -2.2765641, -2.3808427,  4.805343 ],\n",
       "       [-1.6559981,  3.2012947, -2.8166306, -2.6133683,  2.4195454],\n",
       "       [-1.1595987,  0.1654929, -1.5438129,  6.2252   , -2.032078 ],\n",
       "       ...,\n",
       "       [ 7.380243 , -1.9375631, -1.9634055, -1.8967372, -1.0295947],\n",
       "       [ 7.363088 , -1.8969519, -1.9314704, -1.8233254, -1.1142054],\n",
       "       [ 7.381504 , -1.9152062, -1.9537535, -1.8294564, -1.0896233]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "predictions = model.predict(validation_input_ids)   # 輸出的是n*5的編碼值array\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 1, 3, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = np.argmax(predictions, axis=1)         # axis = 1是取行的最大值的索引，0是列的最大值的索引\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9617179088407705\n",
      "[[2409    2    0    7   21]\n",
      " [  19 2175    0    9   70]\n",
      " [   1   19  131    3   24]\n",
      " [   2   13    0  625   18]\n",
      " [  14   96    0    2 2699]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99      2439\n",
      "           1       0.94      0.96      0.95      2273\n",
      "           2       1.00      0.74      0.85       178\n",
      "           3       0.97      0.95      0.96       658\n",
      "           4       0.95      0.96      0.96      2811\n",
      "\n",
      "    accuracy                           0.96      8359\n",
      "   macro avg       0.97      0.92      0.94      8359\n",
      "weighted avg       0.96      0.96      0.96      8359\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "# from sklearn.metrics import precision_score\n",
    "# from sklearn.metrics import recall_score\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.metrics import cohen_kappa_score\n",
    "# from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(accuracy_score(validation_label, predictions))\n",
    "# print(precision_score(validation_label, predictions))\n",
    "# print(recall_score(validation_label, predictions))\n",
    "# print(f1_score(validation_label, predictions))\n",
    "print(confusion_matrix(validation_label, predictions))\n",
    "print(classification_report(validation_label, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}