5.TFBertModel_custom_layer_save.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sbcredFPR_yt"
   },
   "outputs": [],
   "source": [
    "# # google colab tesla P100\n",
    "# ! pip install numpy==1.17.4 scipy==1.3.1 pandas==0.25.3 tensorflow-gpu==2.0.0 torch==1.3.1 torchvision==0.4.2 scikit-learn==0.21.3\n",
    "# ! pip install transformers==2.2.1\n",
    "# ! pip install git+https://github.com/huggingface/transformers.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 836
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2323,
     "status": "ok",
     "timestamp": 1575394763827,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "AyHHIinDR0S4",
    "outputId": "98528166-f55f-45cb-b569-81ecdddea2df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Dec  6 21:47:52 2019       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  GeForce RTX 2080    On   | 00000000:01:00.0  On |                  N/A |\n",
      "| 23%   34C    P8    11W / 215W |     90MiB /  7951MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|    0      1085      G   /usr/lib/xorg/Xorg                            21MiB |\n",
      "|    0      1114      G   /usr/bin/gnome-shell                          67MiB |\n",
      "+-----------------------------------------------------------------------------+\n",
      "\n",
      "Architecture:        x86_64\n",
      "CPU op-mode(s):      32-bit, 64-bit\n",
      "Byte Order:          Little Endian\n",
      "CPU(s):              12\n",
      "On-line CPU(s) list: 0-11\n",
      "Thread(s) per core:  2\n",
      "Core(s) per socket:  6\n",
      "Socket(s):           1\n",
      "NUMA node(s):        1\n",
      "Vendor ID:           GenuineIntel\n",
      "CPU family:          6\n",
      "Model:               158\n",
      "Model name:          Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\n",
      "Stepping:            10\n",
      "CPU MHz:             3975.744\n",
      "CPU max MHz:         4600.0000\n",
      "CPU min MHz:         800.0000\n",
      "BogoMIPS:            6384.00\n",
      "Virtualization:      VT-x\n",
      "L1d cache:           32K\n",
      "L1i cache:           32K\n",
      "L2 cache:            256K\n",
      "L3 cache:            12288K\n",
      "NUMA node0 CPU(s):   0-11\n",
      "Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d\n",
      "\n",
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:            47G        4.7G         36G        237M        5.8G         41G\n",
      "Swap:          2.0G          0B        2.0G\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# # linux系統指令 可省略 win可能跑不了\n",
    "# ! nvidia-smi\n",
    "# ! lscpu\n",
    "# ! free -h\n",
    "try:\n",
    "    import os\n",
    "    f = os.popen('nvidia-smi')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "    f = os.popen('lscpu')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "    f = os.popen('free -h')\n",
    "    f = f.read()\n",
    "    print(f)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ma3jBl3VR0S9"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, TFBertModel, TFBertForSequenceClassification, TFBertMainLayer, AdamW, BertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11967,
     "status": "ok",
     "timestamp": 1575394773623,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "gDBu_JefR0S_",
    "outputId": "bf8e9c94-494d-4197-d351-7dbd1a1e5c73"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>大哥大，透過4G漫遊方式共網爭議，戰火延燒，威寶電信（台灣之星）及中華電信、遠傳電信等業者，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16713</th>\n",
       "      <td>16714</td>\n",
       "      <td>3</td>\n",
       "      <td>上市公司玉晶光（3406）今天傳出裁員，對此，公司發言人趙志強表示，並非裁員而 上市公司玉晶...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16714</th>\n",
       "      <td>16715</td>\n",
       "      <td>3</td>\n",
       "      <td>繼台企銀（2834）工會因合併問題而罷工，創金融史首例後，大眾銀行（2847）工 繼台企銀（...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16715</th>\n",
       "      <td>16716</td>\n",
       "      <td>3</td>\n",
       "      <td>6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16716</th>\n",
       "      <td>16717</td>\n",
       "      <td>3</td>\n",
       "      <td>智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16717</th>\n",
       "      <td>16718</td>\n",
       "      <td>3</td>\n",
       "      <td>時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16718 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  label                                               text\n",
       "0          1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...\n",
       "1          2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...\n",
       "2          3      1  測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...\n",
       "3          4      1  大哥大，透過4G漫遊方式共網爭議，戰火延燒，威寶電信（台灣之星）及中華電信、遠傳電信等業者，...\n",
       "4          5      1  測報告中指出，各款手機在裝上SIM卡前後開機連線時，傳送之資料皆未涉及第一類的敏感資訊，僅為...\n",
       "...      ...    ...                                                ...\n",
       "16713  16714      3  上市公司玉晶光（3406）今天傳出裁員，對此，公司發言人趙志強表示，並非裁員而 上市公司玉晶...\n",
       "16714  16715      3  繼台企銀（2834）工會因合併問題而罷工，創金融史首例後，大眾銀行（2847）工 繼台企銀（...\n",
       "16715  16716      3  6億元賣給英業達，獲利21億元，預計明年首季入帳，活化資產。外界認為這將有機會帶動宏達電明年...\n",
       "16716  16717      3  智慧手機大廠宏達電，要迎接新的一年之前，卻傳出讓人遺憾的消息！員工爆料，有兩位中階主管，傳出...\n",
       "16717  16718      3  時數8到9小時是正常工時。宏達電董事長王雪紅(2012.08.28)：「我們宏達電一向都是從...\n",
       "\n",
       "[16718 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 讀取tsv # \\t tab做區隔\n",
    "df_news = pd.read_csv(\"https://github.com/roccqqck/news_bert/raw/master/data/2015_Company.tsv\", sep=\"\\t\", encoding=\"utf-8\")\n",
    "df_news['text'] = df_news['text'].astype(str)\n",
    "df_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11953,
     "status": "ok",
     "timestamp": 1575394773626,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "5TVWe2l7R0TC",
    "outputId": "2c9ec37d-335e-46f1-dc3e-427dc087f9ac"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    5623\n",
       "0    4877\n",
       "1    4546\n",
       "3    1316\n",
       "2     356\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_ZWHcR_lR0TD"
   },
   "outputs": [],
   "source": [
    "# 文章字數 > 510了話 去尾\n",
    "# 字數小於512-2 因為還有CLS SEP\n",
    "def remove_510(text):\n",
    "    if len(text) > 510:\n",
    "        text = text[:510]    # 只取前510個字\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 111
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11932,
     "status": "ok",
     "timestamp": 1575394773630,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "ED4BgxnRR0TF",
    "outputId": "953467bd-7fd7-49ef-9fcc-e1733e293e19"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...\n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_news[\"text\"] = df_news[\"text\"].apply(remove_510)\n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "u4YvGBxoR0TH"
   },
   "outputs": [],
   "source": [
    "# Load pre-trained model tokenizer, to convert our text into tokens that correspond to BERT’s vocabulary.\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xmbgirEJR0TJ"
   },
   "source": [
    "![avatar](https://github.com/roccqqck/news_bert/raw/master/bert_input_encoding.jpg)\n",
    "\n",
    "https://github.com/roccqqck/news_bert/raw/master/bert_input_encoding.jpg\n",
    "\n",
    "bert input features 有3個\n",
    "\n",
    "input_ids: 代表識別每個 token 的索引值，用 tokenizer 轉換即可\n",
    "\n",
    "token_type_ids: 用來識別句子界限。第一句為 0，第二句則為 1。另外注意句子間的 [SEP] 為 0     (optional) 輸入有1句非必要 輸入有2句則必要\n",
    "\n",
    "attention_mask: 用來界定自注意力機制範圍。1 讓 BERT 關注該位置，0 則代表是 padding 不需關注\n",
    "(optional)\n",
    "\n",
    "https://huggingface.co/transformers/model_doc/bert.html#tfbertforsequenceclassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12319,
     "status": "ok",
     "timestamp": 1575394774044,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "mgmKGfquR0TK",
    "outputId": "c998df77-5fa3-4664-da1c-c03944d77b56"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['很', '好', '看', '的', '動', '作', '片', '，', '不', '會', '浪', '費', '錢',\n",
       "       '跟', '時', '間', '。', '很', '久', '沒', '有', '這', '樣', '的', '探', '險',\n",
       "       '片', '。', '可', '說', '是', '女', '版', '的', '印', '第', '安', '那', '瓊',\n",
       "       '。'], dtype='<U1')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenize input\n",
    "text = \"很好看的動作片，不會浪費錢跟時間。很久沒有這樣的探險片。可說是女版的印第安那瓊。\"\n",
    "tokens = tokenizer.tokenize(text)      # 每個字切詞成一個list\n",
    "print(type(tokens))                 # list\n",
    "np.array(tokens)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12307,
     "status": "ok",
     "timestamp": 1575394774044,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "gVtAD7l-R0TL",
    "outputId": "18abf739-ac5e-43a4-b278-2cc1c31c44db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "40\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298, 3857,\n",
       "       6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300, 6857,\n",
       "       3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957, 4276,\n",
       "       4638, 1313, 5018, 2128, 6929, 4475,  511])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.convert_tokens_to_ids(tokens)   # 每個字轉成id\n",
    "print(type(input_ids))                         # list\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12297,
     "status": "ok",
     "timestamp": 1575394774045,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "PMYLeSnRR0TN",
    "outputId": "8ba9258a-b357-4e56-bee9-089e1a80e6ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids) # token_type_ids 必須input還沒加CLS SEP\n",
    "print(type(token_type_ids))                                # list\n",
    "print(len(token_type_ids)) \n",
    "np.array(token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12285,
     "status": "ok",
     "timestamp": 1575394774046,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "U1G2XkvCR0TP",
    "outputId": "fe545047-23e4-4559-b6f0-8bb6f47a7f4c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "print(type(input_ids))\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 833
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12273,
     "status": "ok",
     "timestamp": 1575394774046,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "4XaqKaQ0R0TR",
    "outputId": "ab116348-ae71-4d2d-dc2d-121c3ead623b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = 512 - len(input_ids)\n",
    "input_ids2 = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "# array右邊append n 個 0  補長度到512\n",
    "print(len(input_ids2))\n",
    "input_ids2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12259,
     "status": "ok",
     "timestamp": 1575394774047,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "gFIOkIB8R0TT",
    "outputId": "6cbfb05c-74c5-4e32-ec99-d4f41e30a6f8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['很', '好', '看', '的', '動', '作', '片'], dtype='<U1')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input如果是兩個句子\n",
    "\n",
    "text = \"很好看的動作片\"\n",
    "tokens = tokenizer.tokenize(text)      # 每個字切詞成一個list\n",
    "print(type(tokens))                 # list\n",
    "np.array(tokens)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12246,
     "status": "ok",
     "timestamp": 1575394774047,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "yJi4wlJYR0TV",
    "outputId": "85bd3fea-6d59-4fe4-e527-a462366da8d5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['不', '會', '浪', '費', '錢', '跟', '時', '間'], dtype='<U1')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text2 = \"不會浪費錢跟時間\"\n",
    "tokens2 = tokenizer.tokenize(text2)      # 每個字切詞成一個list\n",
    "print(type(tokens2))                 # list\n",
    "np.array(tokens2)                    # 轉成numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12237,
     "status": "ok",
     "timestamp": 1575394774048,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "wEbmb0aiR0TW",
    "outputId": "3bcc6713-5d7c-4a85-fe25-793fc698ee38"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([2523, 1962, 4692, 4638, 1240,  868, 4275])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.convert_tokens_to_ids(tokens)   # 每個字轉成id\n",
    "print(type(input_ids))                         # list\n",
    "print(len(input_ids))\n",
    "np.array(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12221,
     "status": "ok",
     "timestamp": 1575394774048,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "iFz7uJXgR0TY",
    "outputId": "a1acecff-0036-411a-a5a4-596180e40d3a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 679, 3298, 3857, 6527, 7092, 6656, 3229, 7279])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids2 = tokenizer.convert_tokens_to_ids(tokens2)   # 每個字轉成id\n",
    "print(type(input_ids2))                         # list\n",
    "print(len(input_ids2))\n",
    "np.array(input_ids2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12211,
     "status": "ok",
     "timestamp": 1575394774049,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "zXo033fGR0Tc",
    "outputId": "a66ae68f-4afb-4693-a9b4-2c7c54a6f267"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids, input_ids2) # token_type_ids 必須input還沒加CLS SEP\n",
    "print(type(token_type_ids))                                # list\n",
    "print(len(token_type_ids)) \n",
    "np.array(token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12201,
     "status": "ok",
     "timestamp": 1575394774049,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "NK9vqFwPR0Te",
    "outputId": "d0d1ec85-327d-402d-d1e8-7ec42ba977ce"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275,  102,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  102])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids3 = tokenizer.build_inputs_with_special_tokens(input_ids, input_ids2)    # 句子前後加上 CLS SEP 的 id\n",
    "print(type(input_ids3))\n",
    "print(len(input_ids3))\n",
    "np.array(input_ids3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yZoA7q-oR0Tg"
   },
   "source": [
    "numpy.pad   補0到某長度\n",
    "\n",
    "https://docs.scipy.org/doc/numpy/reference/generated/numpy.pad.html \n",
    "\n",
    "也可以使用\n",
    "\n",
    "```from keras.preprocessing.sequence import pad_sequences```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UQBhQvBYR0Tg"
   },
   "outputs": [],
   "source": [
    "def input_ids_all(text):\n",
    "#    tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "    tokens = tokenizer.tokenize(text)        # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    if len(input_ids) < 512:\n",
    "        n = 512 - len(input_ids)\n",
    "        input_ids = np.pad(input_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512\n",
    "    return input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 816
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12182,
     "status": "ok",
     "timestamp": 1575394774050,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "Iw86A4PxR0Ti",
    "outputId": "b56e075b-b8b5-49b3-cf8b-10da8ee32add"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 101, 2523, 1962, 4692, 4638, 1240,  868, 4275, 8024,  679, 3298,\n",
       "       3857, 6527, 7092, 6656, 3229, 7279,  511, 2523,  719, 3760, 3300,\n",
       "       6857, 3564, 4638, 2968, 7402, 4275,  511, 1377, 6303, 3221, 1957,\n",
       "       4276, 4638, 1313, 5018, 2128, 6929, 4475,  511,  102,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"很好看的動作片，不會浪費錢跟時間。很久沒有這樣的探險片。可說是女版的印第安那瓊。\"\n",
    "input_ids_all(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2XlNhqFkR0Tl"
   },
   "outputs": [],
   "source": [
    "def attention_mask_all(text):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = tokenizer.build_inputs_with_special_tokens(input_ids)    # 句子前後加上 CLS SEP 的 id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    attention_mask = np.array([1,1])\n",
    "    attention_mask = np.pad(attention_mask, (0, len(input_ids)-2 ), mode ='constant', constant_values=(1)) \n",
    "    # array右邊append 1 到跟segment一樣長\n",
    "    if len(attention_mask) < 512:\n",
    "        n = 512 - len(attention_mask)\n",
    "        attention_mask = np.pad(attention_mask, (0, n), mode ='constant', constant_values=(0))  # array右邊append n 個 0  補長度到512\n",
    "    return attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 425
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12162,
     "status": "ok",
     "timestamp": 1575394774051,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "3hlFBiD4R0Tn",
    "outputId": "da733f1c-7c5d-412d-c3d5-7ab1792a5dab"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask_all(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WF0jLEeaR0Tq"
   },
   "outputs": [],
   "source": [
    "# 事實上單一句子　出來都是0　不做也沒差\n",
    "\n",
    "def token_type_ids_all(text):\n",
    "    tokens = tokenizer.tokenize(text)       # 每個字切詞成一個list\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)  # 每個字轉成id\n",
    "    input_ids = np.array(input_ids)          # list 轉 numpy\n",
    "    token_type_ids = tokenizer.create_token_type_ids_from_sequences(input_ids)   # token_type_ids 必須input還沒加CLS SEP\n",
    "    token_type_ids = np.array(token_type_ids)              # list 轉numpy\n",
    "    if len(token_type_ids) < 512:\n",
    "        n = 512 - len(token_type_ids)\n",
    "        token_type_ids = np.pad(token_type_ids, (0, n), mode ='constant', constant_values=(0))  \n",
    "        # array右邊append n 個 0  補長度到512    \n",
    "    return token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 425
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12145,
     "status": "ok",
     "timestamp": 1575394774052,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "le6Ht-LLR0Ts",
    "outputId": "a96eea71-9d9d-461c-b760-c1f49aa4bbc7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids_all(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CD_KUb5zR0T1"
   },
   "source": [
    "最後我決定用pandas的apply 比較好視覺化理解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 62654,
     "status": "ok",
     "timestamp": 1575394824573,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "KXG8SPFwR0T1",
    "outputId": "b83f149d-8986-4545-fb1d-ea4fefae77f2"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>attention_mask</th>\n",
       "      <th>token_type_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...</td>\n",
       "      <td>[101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...</td>\n",
       "      <td>[101, 5635, 676, 6640, 809, 677, 8024, 519, 17...</td>\n",
       "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                               text  \\\n",
       "0   1      0  財政部充分尊重銀行團確保債權的必要作為，預計元旦後就會請法院實施假扣押，如三重土地順利拍賣，...   \n",
       "1   2      0  至三趴以上，「在假扣押前，仍歡迎頂新還錢。」頂新：將持續協商財政部次長吳當傑尊重並同意銀行團...   \n",
       "\n",
       "                                           input_ids  \\\n",
       "0  [101, 6512, 3124, 6956, 1041, 1146, 2203, 7028...   \n",
       "1  [101, 5635, 676, 6640, 809, 677, 8024, 519, 17...   \n",
       "\n",
       "                                      attention_mask  \\\n",
       "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
       "\n",
       "                                      token_type_ids  \n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df['text2'] = df['text']].apply(lambda x: \" \".join(jieba.cut(x)))\n",
    "# df_news['tokens'] = df_news['text'].apply(lambda x:  tokenizer.tokenize(x) )\n",
    "df_news['input_ids'] = df_news['text'].apply(input_ids_all)\n",
    "df_news['attention_mask'] = df_news['text'].apply(attention_mask_all)\n",
    "df_news['token_type_ids'] = df_news['text'].apply(token_type_ids_all)\n",
    "df_news.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xJ4NflIoR0T3"
   },
   "outputs": [],
   "source": [
    "# df_news['input_ids'].to_numpy()   # 提出來 竟然不是2d numpy 不能這樣做"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 833
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 62639,
     "status": "ok",
     "timestamp": 1575394824576,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "9jp_F9CSR0T5",
    "outputId": "d3973c10-803e-4ab9-d99f-4ecf157fae55"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 101, 6512, 3124, 6956, 1041, 1146, 2203, 7028, 7065, 6121, 1757,\n",
       "       4825,  924, 1002, 3609, 4638, 2553, 6206,  868, 4158, 8024, 7521,\n",
       "       6243, 1039, 3190, 2527, 2218, 3298, 6313, 3791, 7368, 2179, 3177,\n",
       "        969, 2807, 2852, 8024, 1963,  676, 7028, 1759, 1765, 7518, 1164,\n",
       "       2864, 6546, 8024, 7515, 4372, 6917, 3621, 2200,  679, 2512, 7513,\n",
       "       1456, 1059,  127, 1283, 1399, 1519, 2339, 4638, 4495, 6243,  511,\n",
       "       1042, 6493, 7065, 6121, 6134, 4850, 8024, 7515, 4372, 7274, 4634,\n",
       "       5474, 6526, 2234, 3309, 6889, 5147, 3428, 8024, 5195, 5474, 6526,\n",
       "       7065, 6121, 1757, 3748, 6359, 4634, 1139,  998, 1440, 2100, 6349,\n",
       "        928, 1141, 8024, 6313,  102,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['input_ids'][0]))\n",
    "df_news['input_ids'][0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 442
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 62629,
     "status": "ok",
     "timestamp": 1575394824577,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "Zl0pzMg8R0T7",
    "outputId": "ec221a1c-7e64-4bcf-e1d0-c8466f8a01dc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['token_type_ids'][0]))\n",
    "df_news['token_type_ids'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 442
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 62619,
     "status": "ok",
     "timestamp": 1575394824578,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "fG8wEPWFR0T9",
    "outputId": "5aee98b5-0183-41b7-fa55-4010e6765f18"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "512\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_news['attention_mask'][0]))\n",
    "df_news['attention_mask'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-Wt1ZV11R0T_"
   },
   "outputs": [],
   "source": [
    "# # df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# # 只好用for loop一個一個拿出來合併\n",
    "# # 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "\n",
    "# input_ids = np.zeros((1, 512)).astype(int) #宣吿一個都是0的1*512 numpy # np.zeros預設是float 改成int 不然bert餵不進去\n",
    "# for index, row in df_news.iterrows():  \n",
    "#     element = df_news.loc[index,'input_ids']\n",
    "#     input_ids = np.vstack((input_ids, np.array([element])))   # 2維 合併\n",
    "\n",
    "# input_ids = np.delete(input_ids, 0, 0)              # 刪掉一開始都是0的那一個宣告  \n",
    "# input_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dYF0eL2mR0UB"
   },
   "source": [
    "https://www.quora.com/Is-it-better-to-use-np-append-or-list-append"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 64109,
     "status": "ok",
     "timestamp": 1575394826087,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "Hlts4ieIR0UB",
    "outputId": "c8852566-b147-4f55-9b5d-4fc1b473f046"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_news['input_ids'].to_numpy() 出來不是一個2d numpy \n",
    "# 只好用for loop一個一個拿出來合併\n",
    "# 用np vstack超級慢 不知道為何  改用最外層是list append\n",
    "input_ids = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'input_ids']    # 1d np arrary\n",
    "    input_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "input_ids = np.array(input_ids)                       # 轉成2d np\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 64100,
     "status": "ok",
     "timestamp": 1575394826090,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "DcQZspEnR0UC",
    "outputId": "220bd737-ab98-481b-f9f5-0d64c1543c67"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-w33DxRsR0UE"
   },
   "outputs": [],
   "source": [
    "# torch.tensor(input_ids)    # numpy 轉 torch tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 64064,
     "status": "ok",
     "timestamp": 1575394826095,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "Bo2_GES-R0UG",
    "outputId": "9191f5b1-4146-434a-f718-2bde27041014"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=2, shape=(16718, 512), dtype=int64, numpy=\n",
       "array([[ 101, 6512, 3124, ...,    0,    0,    0],\n",
       "       [ 101, 5635,  676, ...,    0,    0,    0],\n",
       "       [ 101, 3947, 1841, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [ 101,  127, 1023, ...,    0,    0,    0],\n",
       "       [ 101, 3255, 2716, ...,    0,    0,    0],\n",
       "       [ 101, 3229, 3149, ...,    0,    0,    0]])>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.convert_to_tensor(input_ids)   # numpy 轉 tf tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 65637,
     "status": "ok",
     "timestamp": 1575394827684,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "_l7qtBTSR0UI",
    "outputId": "8611de65-8074-4fd9-b40a-1ded27b89806"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'attention_mask']    # 1d np arrary\n",
    "    attention_mask.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "attention_mask = np.array(attention_mask)                       # 轉成2d np\n",
    "attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 65626,
     "status": "ok",
     "timestamp": 1575394827686,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "UTKZj-BQR0UK",
    "outputId": "75db2d9b-37bc-443e-868c-20f78d598da3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 67335,
     "status": "ok",
     "timestamp": 1575394829405,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "hvKqAialR0UM",
    "outputId": "cc214aac-2830-4725-f278-e83097d834a4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids = []      # list\n",
    "for index, row in df_news.iterrows():  \n",
    "    np_1d = df_news.loc[index,'token_type_ids']    # 1d np arrary\n",
    "    token_type_ids.append(np_1d)                       # 1d np的 list # list[np_1, np_2, np_3, ....]\n",
    "\n",
    "\n",
    "token_type_ids = np.array(token_type_ids)                       # 轉成2d np\n",
    "token_type_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 67318,
     "status": "ok",
     "timestamp": 1575394829406,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "oF3VFkIwR0UN",
    "outputId": "1f901cc1-33b5-46c0-d898-9aad52b6895a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16718, 512)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_type_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 67305,
     "status": "ok",
     "timestamp": 1575394829407,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "rZwL1gABR0UO",
    "outputId": "aa8fdd12-48df-49de-d364-5bd186577be2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, ..., 3, 3, 3])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label = df_news['label'].to_numpy()\n",
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YyUvJViVR0UT"
   },
   "outputs": [],
   "source": [
    "# 把2個input_ids, attention_mask , token_type_ids 還有label 切成training data, validation data\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Use train_test_split to split our data into train and validation sets for training\n",
    "\n",
    "# # 設定 stratify = label 把每個類別平均\n",
    "train_input_ids, validation_input_ids, train_label, validation_label = train_test_split(input_ids, label, \n",
    "                                                            random_state=2018, test_size=0.5, stratify=label )\n",
    "\n",
    "train_attention_mask, validation_attention_mask, _, _ = train_test_split(attention_mask, label,\n",
    "                                             random_state=2018, test_size=0.5, stratify=label )\n",
    "\n",
    "train_token_type_ids, validation_token_type_ids, _, _ = train_test_split(token_type_ids, label,\n",
    "                                             random_state=2018, test_size=0.5, stratify=label )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EnNTd15KR0UW"
   },
   "outputs": [],
   "source": [
    "# # input 可用 numpy 或 tf tensor 下面是numpy轉tf tensor\n",
    "\n",
    "# train_input_ids = tf.convert_to_tensor(train_input_ids)\n",
    "# validation_input_ids = tf.convert_to_tensor(validation_input_ids)\n",
    "# train_label = tf.convert_to_tensor(train_label)\n",
    "# validation_label = tf.convert_to_tensor(validation_label)\n",
    "# train_attention_mask = tf.convert_to_tensor(train_attention_mask)\n",
    "# validation_attention_mask = tf.convert_to_tensor(validation_attention_mask)\n",
    "# train_token_type_ids = tf.convert_to_tensor(train_token_type_ids)\n",
    "# validation_token_type_ids = tf.convert_to_tensor(validation_token_type_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ThawG-t8R0UX"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import Sequential, Model, Input\n",
    "from tensorflow.keras.layers import Dense, Embedding, SpatialDropout1D, Dropout, Activation, Flatten, InputLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 323
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 79598,
     "status": "ok",
     "timestamp": 1575394841730,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "spvFMporR0UZ",
    "outputId": "9b4d5056-4e4b-4578-8d8f-1a1c11cd13b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 512)]             0         \n",
      "_________________________________________________________________\n",
      "tf_bert_model_1 (TFBertModel ((None, 512, 768), (None, 102267648 \n",
      "_________________________________________________________________\n",
      "dropout_75 (Dropout)         (None, 512, 768)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 393216)            0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 5)                 1966085   \n",
      "=================================================================\n",
      "Total params: 104,233,733\n",
      "Trainable params: 104,233,733\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule \n",
    "# model = TFBertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=5)\n",
    "# 下面model跟 TFBertForSequenceClassification一樣\n",
    "\n",
    "input_layer = Input(shape = (512,), dtype='int64')  # 預設是float 要改成input_id的int64\n",
    "bert = TFBertModel.from_pretrained('bert-base-chinese')(input_layer)\n",
    "bert = bert[0]     # 有bug 修正後可能不需要這行\n",
    "dropout = Dropout(0.1)(bert)\n",
    "flat = Flatten()(dropout)\n",
    "classifier = Dense(units=5, activation=\"softmax\")(flat)                  # 分5類 # sigmoid改softmax\n",
    "model = Model(inputs=input_layer, outputs=classifier)\n",
    "model.summary()\n",
    "\n",
    "# softmax只好用5e-6 # sigmoid可以用1e-5 # 預設無activation了話可以3e-5\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=7e-6, epsilon=1e-08, clipnorm=1.0)\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
    "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 899145,
     "status": "ok",
     "timestamp": 1575395661285,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "zXQWpVruR0Ub",
    "outputId": "716af17f-ebd8-4ebc-cfd3-57bf29e6e422"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8359 samples, validate on 8359 samples\n",
      "Epoch 1/4\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_1/bert/pooler/dense/kernel:0', 'tf_bert_model_1/bert/pooler/dense/bias:0'] when minimizing the loss.\n",
      "8359/8359 [==============================] - 994s 119ms/sample - loss: 1.1737 - accuracy: 0.7296 - val_loss: 1.0194 - val_accuracy: 0.8856\n",
      "Epoch 2/4\n",
      "8359/8359 [==============================] - 987s 118ms/sample - loss: 1.0397 - accuracy: 0.8646 - val_loss: 1.0451 - val_accuracy: 0.8596\n",
      "Epoch 3/4\n",
      "8359/8359 [==============================] - 986s 118ms/sample - loss: 1.0173 - accuracy: 0.8874 - val_loss: 1.0019 - val_accuracy: 0.9029\n",
      "Epoch 4/4\n",
      "8359/8359 [==============================] - 988s 118ms/sample - loss: 1.0175 - accuracy: 0.8868 - val_loss: 0.9935 - val_accuracy: 0.9111\n",
      "CPU times: user 59min 18s, sys: 8min 18s, total: 1h 7min 37s\n",
      "Wall time: 1h 5min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train and evaluate using tf.keras.Model.fit()  # batch size 8就會error 是GPU記憶體爆掉\n",
    "model_fit = model.fit(train_input_ids, train_label, \n",
    "                      batch_size=4, epochs=4, \n",
    "                      validation_data=(validation_input_ids, validation_label)\n",
    "#                      steps_per_epoch=115,\n",
    "#                      validation_steps=7)\n",
    "                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wLF-5C-NR0Uc"
   },
   "source": [
    "https://huggingface.co/transformers/model_doc/bert.html#tfbertforsequenceclassification\n",
    "\n",
    "attention_mask 或 token_type_ids 不一定要放輸入 可選擇\n",
    "\n",
    "要放了話要加```[ ]```\n",
    "\n",
    "```model.fit([train_input_ids, train_attention_mask, train_token_type_ids], train_label)```\n",
    "\n",
    "就是```model.fit(X_train, Y_train)```\n",
    "\n",
    "```[train_input_ids, train_attention_mask, train_token_type_ids]``` 就是 ```X_train```\n",
    "\n",
    "```train_label``` 就是 ```Y_train```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PHMA1WfCR0Ud"
   },
   "outputs": [],
   "source": [
    "# model_fit = model.fit([train_input_ids, train_attention_mask, train_token_type_ids], train_label, \n",
    "#                        batch_size=4, epochs=1, \n",
    "#                     validation_data=([validation_input_ids, validation_attention_mask, validation_token_type_ids], validation_label)\n",
    "#                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YKB4WzGGR0Uf"
   },
   "outputs": [],
   "source": [
    "# # 如果train到一半 想要重新train 在jupyter裡面interrupt kernel\n",
    "# # 這時候 model還是存在在記憶體裡面 只是train到一半 要重新train要釋放model的記憶體\n",
    "# del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9wj2xHOLR0Ug"
   },
   "outputs": [],
   "source": [
    "# model.evaluate(validation_input_ids, validation_label, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SIbipHKaR0Uh"
   },
   "source": [
    "https://blog.csdn.net/zds13257177985/article/details/80638384\n",
    "\n",
    "```predictions = model.predict（test）```預測的是數值，而且輸出的是n*5的編碼值array\n",
    "\n",
    "要經過```predictions = np.argmax(predictions, axis=1)```才是類別"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1074236,
     "status": "ok",
     "timestamp": 1575395836401,
     "user": {
      "displayName": "潘柏瑞",
      "photoUrl": "",
      "userId": "01547540173194355237"
     },
     "user_tz": -480
    },
    "id": "uTZSs6dvR0Ui",
    "outputId": "2dd68d22-725a-4f2a-d04b-ec1f3f99cecb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8359, 5)\n",
      "CPU times: user 1min 1s, sys: 12.9 s, total: 1min 14s\n",
      "Wall time: 3min 3s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.0000000e+00, 0.0000000e+00, 4.5261925e-28, 0.0000000e+00,\n",
       "        1.0000000e+00],\n",
       "       [0.0000000e+00, 0.0000000e+00, 8.9409336e-29, 0.0000000e+00,\n",
       "        1.0000000e+00],\n",
       "       [0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 1.0000000e+00,\n",
       "        0.0000000e+00],\n",
       "       ...,\n",
       "       [1.0000000e+00, 0.0000000e+00, 5.7412344e-35, 0.0000000e+00,\n",
       "        0.0000000e+00],\n",
       "       [1.0000000e+00, 0.0000000e+00, 5.0242792e-35, 0.0000000e+00,\n",
       "        0.0000000e+00],\n",
       "       [1.0000000e+00, 0.0000000e+00, 4.1663540e-35, 0.0000000e+00,\n",
       "        0.0000000e+00]], dtype=float32)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "predictions = model.predict(validation_input_ids)   # 輸出的是n*5的編碼值array\n",
    "print(predictions.shape)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 4, 3, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = np.argmax(predictions, axis=1)         # axis = 1是取行的最大值的索引，0是列的最大值的索引\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9111137695896638\n",
      "[[2415    9    2    0   13]\n",
      " [  21 2091    8    1  152]\n",
      " [   0   28  104    2   44]\n",
      " [  38  107   12  484   17]\n",
      " [  36  214   35    4 2522]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.98      2439\n",
      "           1       0.85      0.92      0.89      2273\n",
      "           2       0.65      0.58      0.61       178\n",
      "           3       0.99      0.74      0.84       658\n",
      "           4       0.92      0.90      0.91      2811\n",
      "\n",
      "    accuracy                           0.91      8359\n",
      "   macro avg       0.87      0.83      0.84      8359\n",
      "weighted avg       0.91      0.91      0.91      8359\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "# from sklearn.metrics import precision_score\n",
    "# from sklearn.metrics import recall_score\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.metrics import cohen_kappa_score\n",
    "# from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(accuracy_score(validation_label, predictions))\n",
    "# print(precision_score(validation_label, predictions))\n",
    "# print(recall_score(validation_label, predictions))\n",
    "# print(f1_score(validation_label, predictions))\n",
    "print(confusion_matrix(validation_label, predictions))\n",
    "print(classification_report(validation_label, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/barry/miniconda3/envs/bertenv/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: model/model_bert_eland_softmax_2/assets\n"
     ]
    }
   ],
   "source": [
    "# model 存起來\n",
    "\n",
    "tf.keras.models.save_model(\n",
    "    model,\n",
    "    \"model/model_bert_eland_softmax_1\",\n",
    "    overwrite=True,\n",
    "    include_optimizer=True,\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QeV7Fj0TUJuQ"
   },
   "outputs": [],
   "source": [
    "# # Save the entire model to a HDF5 file.\n",
    "# # The '.h5' extension indicates that the model shuold be saved to HDF5.\n",
    "# # model 存起來\n",
    "\n",
    "# model.save('model/my_model.h5') # 失敗 tensorflow2.0 bug 修正後可能可以存成h5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         [(None, 512)]             0         \n",
      "_________________________________________________________________\n",
      "model_1 (Model)              (None, 5)                 104233733 \n",
      "=================================================================\n",
      "Total params: 104,233,733\n",
      "Trainable params: 104,233,733\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Recreate the exact same model, including its weights and the optimizer\n",
    "# 讀取存的model\n",
    "input_layer = Input(shape = (512,), dtype='int64')  # 預設是float 要改成input_id的int64 # tensorflow2.0 bug 修正後可能不需要這行\n",
    "load_model = tf.keras.models.load_model('model/model_bert_eland_softmax_1')(input_layer)\n",
    "new_model = Model(inputs=input_layer, outputs=load_model)   # tensorflow2.0 bug 修正後可能不需要這行\n",
    "\n",
    "# Show the model architecture\n",
    "new_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8359, 5)\n",
      "CPU times: user 1min, sys: 13.9 s, total: 1min 14s\n",
      "Wall time: 3min 2s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.0000000e+00, 0.0000000e+00, 4.5261925e-28, 0.0000000e+00,\n",
       "        1.0000000e+00],\n",
       "       [0.0000000e+00, 0.0000000e+00, 8.9409336e-29, 0.0000000e+00,\n",
       "        1.0000000e+00],\n",
       "       [0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 1.0000000e+00,\n",
       "        0.0000000e+00],\n",
       "       ...,\n",
       "       [1.0000000e+00, 0.0000000e+00, 5.7412344e-35, 0.0000000e+00,\n",
       "        0.0000000e+00],\n",
       "       [1.0000000e+00, 0.0000000e+00, 5.0242792e-35, 0.0000000e+00,\n",
       "        0.0000000e+00],\n",
       "       [1.0000000e+00, 0.0000000e+00, 4.1663540e-35, 0.0000000e+00,\n",
       "        0.0000000e+00]], dtype=float32)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "predictions = new_model.predict(validation_input_ids)   # 輸出的是n*5的編碼值array\n",
    "print(predictions.shape)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 4, 3, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = np.argmax(predictions, axis=1)         # axis = 1是取行的最大值的索引，0是列的最大值的索引\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9111137695896638\n",
      "[[2415    9    2    0   13]\n",
      " [  21 2091    8    1  152]\n",
      " [   0   28  104    2   44]\n",
      " [  38  107   12  484   17]\n",
      " [  36  214   35    4 2522]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.98      2439\n",
      "           1       0.85      0.92      0.89      2273\n",
      "           2       0.65      0.58      0.61       178\n",
      "           3       0.99      0.74      0.84       658\n",
      "           4       0.92      0.90      0.91      2811\n",
      "\n",
      "    accuracy                           0.91      8359\n",
      "   macro avg       0.87      0.83      0.84      8359\n",
      "weighted avg       0.91      0.91      0.91      8359\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "# from sklearn.metrics import precision_score\n",
    "# from sklearn.metrics import recall_score\n",
    "# from sklearn.metrics import f1_score\n",
    "# from sklearn.metrics import cohen_kappa_score\n",
    "# from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(accuracy_score(validation_label, predictions))\n",
    "# print(precision_score(validation_label, predictions))\n",
    "# print(recall_score(validation_label, predictions))\n",
    "# print(f1_score(validation_label, predictions))\n",
    "print(confusion_matrix(validation_label, predictions))\n",
    "print(classification_report(validation_label, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of 4.TFBertModel_custom_layer.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}