4.TFBertModel_custom_layer_output_layer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Nov 27 15:42:26 2019       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  GeForce RTX 208...  On   | 00000000:01:00.0  On |                  N/A |\n",
      "|  0%   49C    P8    12W / 300W |     90MiB / 10988MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|    0      1750      G   /usr/lib/xorg/Xorg                            21MiB |\n",
      "|    0      2004      G   /usr/bin/gnome-shell                          67MiB |\n",
      "+-----------------------------------------------------------------------------+\n",
      "\n",
      "Architecture:        x86_64\n",
      "CPU op-mode(s):      32-bit, 64-bit\n",
      "Byte Order:          Little Endian\n",
      "CPU(s):              12\n",
      "On-line CPU(s) list: 0-11\n",
      "Thread(s) per core:  2\n",
      "Core(s) per socket:  6\n",
      "Socket(s):           1\n",
      "NUMA node(s):        1\n",
      "Vendor ID:           GenuineIntel\n",
      "CPU family:          6\n",
      "Model:               158\n",
      "Model name:          Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\n",
      "Stepping:            10\n",
      "CPU MHz:             3714.806\n",
      "CPU max MHz:         4600.0000\n",
      "CPU min MHz:         800.0000\n",
      "BogoMIPS:            6384.00\n",
      "Virtualization:      VT-x\n",
      "L1d cache:           32K\n",
      "L1i cache:           32K\n",
      "L2 cache:            256K\n",
      "L3 cache:            12288K\n",
      "NUMA node0 CPU(s):   0-11\n",
      "Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d\n",
      "\n",
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:            47G        1.5G         43G        1.7M        1.7G         45G\n",
      "Swap:          2.0G        5.5M        2.0G\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# # linux系統指令 可省略 win可能跑不了\n",
    "# ! nvidia-smi\n",
    "# ! lscpu\n",
    "# ! free -h\n",
    "try:\n",
    "    import os\n",
    "    f = os.popen('nvidia-smi')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "    f = os.popen('lscpu')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "    f = os.popen('free -h')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, TFBertModel, TFBertForSequenceClassification, TFBertMainLayer, AdamW, BertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>大哥大，透過4G漫遊方式共網爭議，戰火延燒，威寶電信（台灣之星）及中華電信、遠傳電信等業者，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16713</th>\n",
       "      <td>16714</td>\n",
       "      <td>3</td>\n",
       "      <td>上市公司玉晶光（3406）今天傳出裁員，對此，公司發言人趙志強表示，並非裁員而 上市公司玉晶...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16714</th>\n",
       "      <td>16715</td>\n",
       "      <td>3</td>\n",
       "      <td>繼台企銀（2834）工會因合併問題而罷工，創金融史首例後，大眾銀行（2847）工 繼台企銀（...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16715</th>\n",
       "      <td>16716</td>\n",
       "      <td>3</td>\n",
       "      <td>6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16716</th>\n",
       "      <td>16717</td>\n",
       "      <td>3</td>\n",
       "      <td>智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16717</th>\n",
       "      <td>16718</td>\n",
       "      <td>3</td>\n",
       "      <td>時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16718 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  label                                               text\n",
       "0          1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...\n",
       "1          2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...\n",
       "2          3      1  測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...\n",
       "3          4      1  大哥大，透過4G漫遊方式共網爭議，戰火延燒，威寶電信（台灣之星）及中華電信、遠傳電信等業者，...\n",
       "4          5      1  測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...\n",
       "...      ...    ...                                                ...\n",
       "16713  16714      3  上市公司玉晶光（3406）今天傳出裁員，對此，公司發言人趙志強表示，並非裁員而 上市公司玉晶...\n",
       "16714  16715      3  繼台企銀（2834）工會因合併問題而罷工，創金融史首例後，大眾銀行（2847）工 繼台企銀（...\n",
       "16715  16716      3  6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年...\n",
       "16716  16717      3  智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出...\n",
       "16717  16718      3  時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從...\n",
       "\n",
       "[16718 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 讀取tsv # \\t tab做區隔\n",
    "df_news = pd.read_csv(\"https://github.com/roccqqck/news_bert/raw/master/data/2015_Company.tsv\", sep=\"\\t\", encoding=\"utf-8\")\n",
    "df_news['text'] = df_news['text'].astype(str)\n",
    "df_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    5623\n",
       "0    4877\n",
       "1    4546\n",
       "3    1316\n",
       "2     356\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文章字數 > 510了話 去尾\n",
    "# 字數小於512-2 因為還有CLS SEP\n",
    "def remove_510(text):\n",
    "    if len(text) > 510:\n",
    "        text = text[:510]    # 只取前510個字\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...\n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news[\"text\"] = df_news[\"text\"].apply(remove_510)\n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pre-trained model tokenizer, to convert our text into tokens that correspond to BERT’s vocabulary.\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![avatar](https://github.com/roccqqck/news_bert/raw/master/bert_input_encoding.jpg)\n",
    "\n",
    "https://github.com/roccqqck/news_bert/raw/master/bert_input_encoding.jpg\n",
    "\n",
    "bert input features 有3個\n",
    "\n",
    "input_ids: 代表識別每個 token 的索引值，用 tokenizer 轉換即可\n",
    "\n",
    "token_type_ids: 用來識別句子界限。第一句為 0，第二句則為 1。另外注意句子間的 [SEP] 為 0     (optional) 輸入有1句非必要 輸入有2句則必要\n",
    "\n",
    "attention_mask: 用來界定自注意力機制範圍。1 讓 BERT 關注該位置，0 則代表是 padding 不需關注\n",
    "(optional)\n",
    "\n",
    "https://huggingface.co/transformers/model_doc/bert.html#tfbertforsequenceclassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['很', '好', '看', '的', '動', '作', '片', '，', '不', '會', '浪', '費', '錢',\n",
       "       '跟', '時', '間', '。', '很', '久', '沒', '有', '這', '樣', '的', '探', '險',\n",
       "       '片', '。', '可', '說', '是', '女', '版', '的', '印', '第', '安', '那', '瓊',\n",
       "       '。'], dtype='<U1')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenize input\n",
    "text = \"很好看的動作片，不會浪費錢跟時間。很久沒有這樣的探險片。可說是女版的印第安那瓊。\"\n",
    "tokens = tokenizer.tokenize(text)      # 每個字切詞成一個list\n",
    "print(type(tokens))                 # list\n",
    "np.array(tokens)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "40\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298, 3857,\n",
       "       6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300, 6857,\n",
       "       3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957, 4276,\n",
       "       4638, 1313, 5018, 2128, 6929, 4475,  511])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.convert_tokens_to_ids(tokens)   # 每個字轉成id\n",
    "print(type(input_ids))                         # list\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids) # token_type_ids 必須input還沒加CLS SEP\n",
    "print(type(token_type_ids))                                # list\n",
    "print(len(token_type_ids)) \n",
    "np.array(token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "print(type(input_ids))\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = 512 - len(input_ids)\n",
    "input_ids2 = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "# array右邊append n 個 0  補長度到512\n",
    "print(len(input_ids2))\n",
    "input_ids2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['很', '好', '看', '的', '動', '作', '片'], dtype='<U1')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input如果是兩個句子\n",
    "\n",
    "text = \"很好看的動作片\"\n",
    "tokens = tokenizer.tokenize(text)      # 每個字切詞成一個list\n",
    "print(type(tokens))                 # list\n",
    "np.array(tokens)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['不', '會', '浪', '費', '錢', '跟', '時', '間'], dtype='<U1')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text2 = \"不會浪費錢跟時間\"\n",
    "tokens2 = tokenizer.tokenize(text2)      # 每個字切詞成一個list\n",
    "print(type(tokens2))                 # list\n",
    "np.array(tokens2)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2523, 1962, 4692, 4638, 1240,  868, 4275])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.convert_tokens_to_ids(tokens)   # 每個字轉成id\n",
    "print(type(input_ids))                         # list\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 679, 3298, 3857, 6527, 7092, 6656, 3229, 7279])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids2 = tokenizer.convert_tokens_to_ids(tokens2)   # 每個字轉成id\n",
    "print(type(input_ids2))                         # list\n",
    "print(len(input_ids2))\n",
    "np.array(input_ids2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids, input_ids2) # token_type_ids 必須input還沒加CLS SEP\n",
    "print(type(token_type_ids))                                # list\n",
    "print(len(token_type_ids)) \n",
    "np.array(token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275,  102,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  102])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids3 = tokenizer.build_inputs_with_special_tokens(input_ids, input_ids2)    # 句子前後加上 CLS SEP 的 id\n",
    "print(type(input_ids3))\n",
    "print(len(input_ids3))\n",
    "np.array(input_ids3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "numpy.pad   補0到某長度\n",
    "\n",
    "https://docs.scipy.org/doc/numpy/reference/generated/numpy.pad.html \n",
    "\n",
    "也可以使用\n",
    "\n",
    "```from keras.preprocessing.sequence import pad_sequences```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def input_ids_all(text):\n",
    "#    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "    tokens = tokenizer.tokenize(text)        # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    if len(input_ids) < 512:\n",
    "        n = 512 - len(input_ids)\n",
    "        input_ids = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512\n",
    "    return input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"很好看的動作片，不會浪費錢跟時間。很久沒有這樣的探險片。可說是女版的印第安那瓊。\"\n",
    "input_ids_all(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attention_mask_all(text):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    attention_mask = np.array([1,1])\n",
    "    attention_mask = np.pad(attention_mask, (0, len(input_ids)-2 ), mode ='constant', constant_values=(1)) \n",
    "    # array右邊append 1 到跟segment一樣長\n",
    "    if len(attention_mask) < 512:\n",
    "        n = 512 - len(attention_mask)\n",
    "        attention_mask = np.pad(attention_mask, (0, n), mode ='constant', constant_values=(0))  # array右邊append n 個 0  補長度到512\n",
    "    return attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask_all(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 事實上單一句子　出來都是0　不做也沒差\n",
    "\n",
    "def token_type_ids_all(text):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids)   # token_type_ids 必須input還沒加CLS SEP\n",
    "    token_type_ids = np.array(token_type_ids)              # list 轉numpy\n",
    "    if len(token_type_ids) < 512:\n",
    "        n = 512 - len(token_type_ids)\n",
    "        token_type_ids = np.pad(token_type_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512    \n",
    "    return token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids_all(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最後我決定用pandas的apply 比較好視覺化理解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>attention_mask</th>\n",
       "      <th>token_type_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>[101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>[101, 5635, 676, 6640, 809, 677, 8024, 519, 17...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text  \\\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   \n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...   \n",
       "\n",
       "                                           input_ids  \\\n",
       "0  [101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...   \n",
       "1  [101, 5635, 676, 6640, 809, 677, 8024, 519, 17...   \n",
       "\n",
       "                                      attention_mask  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                      token_type_ids  \n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df['text2'] = df['text']].apply(lambda x: \" \".join(jieba.cut(x)))\n",
    "# df_news['tokens'] = df_news['text'].apply(lambda x:  tokenizer.tokenize(x) )\n",
    "df_news['input_ids'] = df_news['text'].apply(input_ids_all)\n",
    "df_news['attention_mask'] = df_news['text'].apply(attention_mask_all)\n",
    "df_news['token_type_ids'] = df_news['text'].apply(token_type_ids_all)\n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_news['input_ids'].to_numpy()   # 提出來 竟然不是2d numpy 不能這樣做"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 6512, 3124, 6956, 1041, 1146, 2203, 7028, 7065, 6121, 1757,\n",
       "       4825,  924, 1002, 3609, 4638, 2553, 6206,  868, 4158, 8024, 7521,\n",
       "       6243, 1039, 3190, 2527, 2218, 3298, 6313, 3791, 7368, 2179, 3177,\n",
       "        969, 2807, 2852, 8024, 1963,  676, 7028, 1759, 1765, 7518, 1164,\n",
       "       2864, 6546, 8024, 7515, 4372, 6917, 3621, 2200,  679, 2512, 7513,\n",
       "       1456, 1059,  127, 1283, 1399, 1519, 2339, 4638, 4495, 6243,  511,\n",
       "       1042, 6493, 7065, 6121, 6134, 4850, 8024, 7515, 4372, 7274, 4634,\n",
       "       5474, 6526, 2234, 3309, 6889, 5147, 3428, 8024, 5195, 5474, 6526,\n",
       "       7065, 6121, 1757, 3748, 6359, 4634, 1139,  998, 1440, 2100, 6349,\n",
       "        928, 1141, 8024, 6313,  102,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['input_ids'][0]))\n",
    "df_news['input_ids'][0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['token_type_ids'][0]))\n",
    "df_news['token_type_ids'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['attention_mask'][0]))\n",
    "df_news['attention_mask'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# # 只好用for loop一個一個拿出來合併\n",
    "# # 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "\n",
    "# input_ids = np.zeros((1, 512)).astype(int) #宣吿一個都是0的1*512 numpy # np.zeros預設是float 改成int 不然bert餵不進去\n",
    "# for index, row in df_news.iterrows():  \n",
    "#     element = df_news.loc[index,'input_ids']\n",
    "#     input_ids = np.vstack((input_ids, np.array([element])))   # 2維 合併\n",
    "\n",
    "# input_ids = np.delete(input_ids, 0, 0)              # 刪掉一開始都是0的那一個宣告  \n",
    "# input_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.quora.com/Is-it-better-to-use-np-append-or-list-append"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# 只好用for loop一個一個拿出來合併\n",
    "# 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "input_ids = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'input_ids']    # 1d np arrary\n",
    "    input_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "input_ids = np.array(input_ids)                       # 轉成2d np\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.tensor(input_ids)    # numpy 轉 torch tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=2, shape=(16718, 512), dtype=int64, numpy=\n",
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.convert_to_tensor(input_ids)   # numpy 轉 tf tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'attention_mask']    # 1d np arrary\n",
    "    attention_mask.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "attention_mask = np.array(attention_mask)                       # 轉成2d np\n",
    "attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'token_type_ids']    # 1d np arrary\n",
    "    token_type_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "token_type_ids = np.array(token_type_ids)                       # 轉成2d np\n",
    "token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, ..., 3, 3, 3])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label = df_news['label'].to_numpy()\n",
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把2個input_ids, attention_mask , token_type_ids 還有label 切成training data, validation data\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Use train_test_split to split our data into train and validation sets for training\n",
    "\n",
    "# # 設定 stratify = label 把每個類別平均\n",
    "train_input_ids, validation_input_ids, train_label, validation_label = train_test_split(input_ids, label, \n",
    "                                                            random_state=2018, test_size=0.5, stratify=label )\n",
    "\n",
    "train_attention_mask, validation_attention_mask, _, _ = train_test_split(attention_mask, label,\n",
    "                                             random_state=2018, test_size=0.5, stratify=label )\n",
    "\n",
    "train_token_type_ids, validation_token_type_ids, _, _ = train_test_split(token_type_ids, label,\n",
    "                                             random_state=2018, test_size=0.5, stratify=label )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # input 可用 numpy 或 tf tensor 下面是numpy轉tf tensor\n",
    "\n",
    "# train_input_ids = tf.convert_to_tensor(train_input_ids)\n",
    "# validation_input_ids = tf.convert_to_tensor(validation_input_ids)\n",
    "# train_label = tf.convert_to_tensor(train_label)\n",
    "# validation_label = tf.convert_to_tensor(validation_label)\n",
    "# train_attention_mask = tf.convert_to_tensor(train_attention_mask)\n",
    "# validation_attention_mask = tf.convert_to_tensor(validation_attention_mask)\n",
    "# train_token_type_ids = tf.convert_to_tensor(train_token_type_ids)\n",
    "# validation_token_type_ids = tf.convert_to_tensor(validation_token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, Embedding, SpatialDropout1D, Dropout, Activation, Input, Flatten, InputLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 512)]             0         \n",
      "_________________________________________________________________\n",
      "tf_bert_model (TFBertModel)  ((None, 512, 768), (None, 102267648 \n",
      "_________________________________________________________________\n",
      "dropout_37 (Dropout)         (None, 512, 768)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 393216)            0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 5)                 1966085   \n",
      "=================================================================\n",
      "Total params: 104,233,733\n",
      "Trainable params: 104,233,733\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule \n",
    "# model = TFBertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=5)\n",
    "# 下面model跟 TFBertForSequenceClassification一樣\n",
    "\n",
    "input_layer = Input(shape = (512,), dtype='int64')  # 預設是float 要改成input_id的int64\n",
    "bert = TFBertModel.from_pretrained('bert-base-chinese')(input_layer)\n",
    "bert = bert[0]     # 有bug 修正後可能不需要這行\n",
    "dropout = Dropout(0.1)(bert)\n",
    "flat = Flatten()(dropout)\n",
    "classifier = Dense(units=5)(flat)                  # 分5類\n",
    "model = Model(inputs=input_layer, outputs=classifier)\n",
    "model.summary()\n",
    "\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8359 samples, validate on 8359 samples\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model/bert/pooler/dense/kernel:0', 'tf_bert_model/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "8359/8359 [==============================] - 773s 92ms/sample - loss: 0.3862 - accuracy: 0.8923 - val_loss: 0.3362 - val_accuracy: 0.9450\n",
      "CPU times: user 13min 18s, sys: 1min 36s, total: 14min 55s\n",
      "Wall time: 12min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train and evaluate using tf.keras.Model.fit()  # batch size 8就會error 是GPU記憶體爆掉\n",
    "model_fit = model.fit(train_input_ids, train_label, \n",
    "                      batch_size=4, epochs=1, \n",
    "                      validation_data=(validation_input_ids, validation_label)\n",
    "#                      steps_per_epoch=115,\n",
    "#                      validation_steps=7)\n",
    "                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://huggingface.co/transformers/model_doc/bert.html#tfbertforsequenceclassification\n",
    "\n",
    "attention_mask 或 token_type_ids 不一定要放輸入 可選擇\n",
    "\n",
    "要放了話要加```[ ]```\n",
    "\n",
    "```model.fit([train_input_ids, train_attention_mask, train_token_type_ids], train_label)```\n",
    "\n",
    "就是```model.fit(X_train, Y_train)```\n",
    "\n",
    "```[train_input_ids, train_attention_mask, train_token_type_ids]``` 就是 ```X_train```\n",
    "\n",
    "```train_label``` 就是 ```Y_train```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_fit = model.fit([train_input_ids, train_attention_mask, train_token_type_ids], train_label, \n",
    "#                        batch_size=4, epochs=1, \n",
    "#                     validation_data=([validation_input_ids, validation_attention_mask, validation_token_type_ids], validation_label)\n",
    "#                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 如果train到一半 想要重新train 在jupyter裡面interrupt kernel\n",
    "# # 這時候 model還是存在在記憶體裡面 只是train到一半 要重新train要釋放model的記憶體\n",
    "# del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.evaluate(validation_input_ids, validation_label, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://blog.csdn.net/zds13257177985/article/details/80638384\n",
    "\n",
    "```predictions = model.predict（test）```預測的是數值，而且輸出的是n*5的編碼值array\n",
    "\n",
    "要經過```predictions = np.argmax(predictions, axis=1)```才是類別"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8359, 5)\n",
      "CPU times: user 53.1 s, sys: 7.26 s, total: 1min\n",
      "Wall time: 2min 4s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-3.5264685 ,  1.1058946 , -3.736976  , -1.7348151 ,  7.368966  ],\n",
       "       [-2.375975  ,  2.4117017 , -1.6232955 , -1.5841885 ,  3.160581  ],\n",
       "       [-1.4852978 ,  0.54937685, -3.4269652 ,  1.8822311 ,  1.0742552 ],\n",
       "       ...,\n",
       "       [12.465729  , -5.1978993 , -3.6832724 , -1.7571833 , -2.900733  ],\n",
       "       [13.865915  , -5.704613  , -5.4526525 , -2.153188  , -3.535213  ],\n",
       "       [11.619608  , -4.349374  , -4.1693845 , -2.8048012 , -2.0813828 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "predictions = model.predict(validation_input_ids)   # 輸出的是n*5的編碼值array\n",
    "print(predictions.shape)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 4, 3, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = np.argmax(predictions, axis=1)         # axis = 1是取行的最大值的索引，0是列的最大值的索引\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9292977628902979\n",
      "[[2369   12    6   26   26]\n",
      " [  42 2058   77   26   70]\n",
      " [   0    1  172    0    5]\n",
      " [   9   46   10  559   34]\n",
      " [  16  161   21    3 2610]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.97      0.97      2439\n",
      "           1       0.90      0.91      0.90      2273\n",
      "           2       0.60      0.97      0.74       178\n",
      "           3       0.91      0.85      0.88       658\n",
      "           4       0.95      0.93      0.94      2811\n",
      "\n",
      "    accuracy                           0.93      8359\n",
      "   macro avg       0.87      0.92      0.89      8359\n",
      "weighted avg       0.93      0.93      0.93      8359\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "# from sklearn.metrics import precision_score\n",
    "# from sklearn.metrics import recall_score\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.metrics import cohen_kappa_score\n",
    "# from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(accuracy_score(validation_label, predictions))\n",
    "# print(precision_score(validation_label, predictions))\n",
    "# print(recall_score(validation_label, predictions))\n",
    "# print(f1_score(validation_label, predictions))\n",
    "print(confusion_matrix(validation_label, predictions))\n",
    "print(classification_report(validation_label, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "(8359, 512, 768)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[-8.35068226e-01,  2.32662261e-02,  2.40469247e-01, ...,\n",
       "         -2.58911997e-01, -4.34102714e-01, -6.14368498e-01],\n",
       "        [-1.30837590e-01, -1.29651800e-01,  2.89526135e-02, ...,\n",
       "          2.99261719e-01,  2.57718951e-01, -1.84120968e-01],\n",
       "        [ 5.57101548e-01,  3.96360070e-01,  3.46481860e-01, ...,\n",
       "         -2.87417293e-01,  5.34755766e-01, -3.55959743e-01],\n",
       "        ...,\n",
       "        [-1.01705827e-01, -1.36546753e-02,  7.95840397e-02, ...,\n",
       "          9.30070877e-04,  4.26345542e-02, -1.08621798e-01],\n",
       "        [-8.96287858e-02, -3.12647596e-03,  7.13456273e-02, ...,\n",
       "          1.37357414e-02,  5.16944155e-02, -1.22056179e-01],\n",
       "        [-8.49680752e-02, -2.82232985e-02,  4.34723198e-02, ...,\n",
       "         -5.11656702e-02,  6.36745468e-02, -1.17881253e-01]],\n",
       "\n",
       "       [[-6.30940437e-01,  2.06126630e-01,  1.07969210e-01, ...,\n",
       "          1.23910904e-01, -5.12177467e-01, -3.70369941e-01],\n",
       "        [-2.95967013e-01, -1.58068866e-01,  2.00787187e-01, ...,\n",
       "          7.48085231e-02, -1.70719296e-01, -1.88389406e-01],\n",
       "        [-1.95255756e-01, -2.34043732e-01, -1.15590118e-01, ...,\n",
       "          8.10086131e-02, -1.26870424e-01, -5.47735766e-02],\n",
       "        ...,\n",
       "        [-7.29800910e-02,  2.61030272e-02,  1.29101872e-01, ...,\n",
       "          1.00053236e-01, -1.19802743e-01, -4.57206517e-02],\n",
       "        [-5.52150235e-02,  3.15606073e-02,  1.14747636e-01, ...,\n",
       "          1.00153834e-01, -1.10502273e-01, -6.12954386e-02],\n",
       "        [-5.32841422e-02,  1.99259669e-02,  8.88063759e-02, ...,\n",
       "          2.90653706e-02, -1.01753481e-01, -5.63550517e-02]],\n",
       "\n",
       "       [[ 2.73553804e-02, -2.99750686e-01, -4.58522797e-01, ...,\n",
       "         -1.12304831e+00,  1.34919733e-01, -6.88543797e-01],\n",
       "        [ 1.00807905e-01, -3.11846823e-01, -5.55769563e-01, ...,\n",
       "          6.69205189e-03,  1.81801617e-01, -2.93523252e-01],\n",
       "        [-5.30240417e-01,  3.00621763e-02, -3.42232466e-01, ...,\n",
       "          1.20274246e-01,  7.00019076e-02, -4.31188017e-01],\n",
       "        ...,\n",
       "        [ 1.59684494e-01, -1.15180366e-01, -4.70839381e-01, ...,\n",
       "         -4.04301673e-01,  2.63548285e-01, -4.41732287e-01],\n",
       "        [ 1.98548868e-01, -1.42279014e-01, -4.60208416e-01, ...,\n",
       "         -3.01502019e-01,  2.76499599e-01, -4.47066635e-01],\n",
       "        [ 1.23360954e-01, -8.60320851e-02, -4.24692690e-01, ...,\n",
       "         -4.49277848e-01,  1.63959473e-01, -4.78856534e-01]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[ 3.17584217e-01,  2.16659397e-01,  4.38922465e-01, ...,\n",
       "         -2.52824873e-01,  1.12298578e-02, -6.40907407e-01],\n",
       "        [ 4.29105163e-01,  5.45457155e-02, -2.02625960e-01, ...,\n",
       "          1.17869154e-01, -4.06505406e-01, -5.33706784e-01],\n",
       "        [ 3.80735427e-01, -7.68627599e-02, -5.20156026e-01, ...,\n",
       "          2.68062234e-01, -3.97276908e-01, -6.60961449e-01],\n",
       "        ...,\n",
       "        [ 6.88043684e-02,  2.25396201e-01,  3.19684967e-02, ...,\n",
       "          2.93833375e-01, -3.65255088e-01, -1.89989626e-01],\n",
       "        [ 6.60314262e-02,  2.07611293e-01,  2.16103196e-02, ...,\n",
       "          2.89889604e-01, -3.75589401e-01, -1.98522776e-01],\n",
       "        [ 9.92238969e-02,  2.42610574e-01, -1.49458572e-02, ...,\n",
       "          3.03046197e-01, -3.59082222e-01, -2.04617426e-01]],\n",
       "\n",
       "       [[ 2.78659403e-01,  6.64266869e-02,  4.90318120e-01, ...,\n",
       "         -5.29422462e-02, -1.53791845e-01, -4.85179961e-01],\n",
       "        [ 1.23207115e-01,  5.24559915e-01, -2.84776688e-02, ...,\n",
       "         -2.21202075e-02, -2.25580037e-01,  1.64061248e-01],\n",
       "        [ 2.79503345e-01,  1.82398334e-01, -2.66600549e-02, ...,\n",
       "          3.33776832e-01, -6.37458980e-01, -1.50484055e-01],\n",
       "        ...,\n",
       "        [ 4.54330370e-02,  1.73835978e-01, -1.54539272e-02, ...,\n",
       "          2.55990237e-01, -3.95541102e-01, -1.99050933e-01],\n",
       "        [ 1.19225495e-01,  9.36972946e-02, -1.74645349e-01, ...,\n",
       "          3.02612454e-01, -5.05170822e-01, -3.56516153e-01],\n",
       "        [ 2.62229890e-02,  1.80609256e-01, -1.61288753e-02, ...,\n",
       "          2.41340131e-01, -3.78924668e-01, -2.06494585e-01]],\n",
       "\n",
       "       [[ 2.94005305e-01, -8.97563994e-04,  7.10698724e-01, ...,\n",
       "         -2.04550803e-01, -6.14230037e-02, -5.07999003e-01],\n",
       "        [-3.46598625e-02,  2.55385578e-01,  6.75736219e-02, ...,\n",
       "          2.95261592e-01, -3.68818969e-01, -2.74380952e-01],\n",
       "        [-2.46929318e-01,  7.34514967e-02, -1.61911547e-01, ...,\n",
       "          4.01500553e-01, -3.40034872e-01, -4.82804552e-02],\n",
       "        ...,\n",
       "        [ 7.08978996e-02,  2.26698250e-01, -2.39976645e-02, ...,\n",
       "          2.29500055e-01, -4.24162567e-01, -2.06691891e-01],\n",
       "        [ 5.83084226e-02,  1.99845165e-01, -7.63177499e-02, ...,\n",
       "          2.60780305e-01, -4.44936663e-01, -2.53773242e-01],\n",
       "        [ 6.69618025e-02,  2.49903947e-01, -2.28697583e-02, ...,\n",
       "          2.30418921e-01, -4.19872999e-01, -2.04022825e-01]]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "bert_layer_model1 = Model(inputs=model.input, outputs=model.get_layer('tf_bert_model').output)\n",
    "predictions = bert_layer_model1.predict(validation_input_ids)   # 輸出的是n*5的編碼值array\n",
    "print(type(predictions))\n",
    "print(predictions[0].shape)\n",
    "predictions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "(8359, 393216)\n",
      "CPU times: user 3min 34s, sys: 2min 16s, total: 5min 51s\n",
      "Wall time: 2min 17s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-0.03531556, -0.0270932 ,  0.05939125, ..., -0.00097317,\n",
       "        -0.08195499, -0.01073981],\n",
       "       [-0.02595161, -0.01654707,  0.02728941, ...,  0.02482173,\n",
       "        -0.07865668,  0.01810846],\n",
       "       [ 0.02304554, -0.02106418, -0.15899064, ...,  0.04300633,\n",
       "        -0.145352  , -0.1249648 ],\n",
       "       ...,\n",
       "       [ 0.20038779,  0.17067742,  0.14136411, ...,  0.11787441,\n",
       "        -0.19624126, -0.03155544],\n",
       "       [ 0.21668512,  0.20701514,  0.20434818, ...,  0.21140781,\n",
       "        -0.30936435, -0.00779616],\n",
       "       [ 0.14715588,  0.12922615,  0.25229585, ...,  0.14122306,\n",
       "        -0.19088547,  0.00668337]], dtype=float32)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "flatten_layer_model = Model(inputs=model.input, outputs=model.get_layer('flatten').output)\n",
    "predictions = flatten_layer_model.predict(validation_input_ids)   # 輸出的是n*5的編碼值array\n",
    "print(type(predictions))\n",
    "print(predictions.shape)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}