sbert-mpnet-fb-post.ipynb

{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nfrom matplotlib import pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import precision_recall_curve\n\nimport pickle\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport os\n\nimport string\n\nimport tensorflow_hub as hub\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom numpy import dstack\nfrom sklearn.linear_model import LogisticRegression\n","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:34:58.488420Z","iopub.execute_input":"2023-04-10T04:34:58.488820Z","iopub.status.idle":"2023-04-10T04:35:08.665376Z","shell.execute_reply.started":"2023-04-10T04:34:58.488786Z","shell.execute_reply":"2023-04-10T04:35:08.664285Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"df = pd.read_table('/kaggle/input/fb-post-data/FB_posts_labeled.txt')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:35:08.667472Z","iopub.execute_input":"2023-04-10T04:35:08.668338Z","iopub.status.idle":"2023-04-10T04:35:08.745610Z","shell.execute_reply.started":"2023-04-10T04:35:08.668288Z","shell.execute_reply":"2023-04-10T04:35:08.744640Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"df['text'] = df['message'].map(lambda text: text.translate(str.maketrans('','',string.punctuation)).strip())\ndf['text'] = df['text'].map(lambda text: text.translate(str.maketrans('','',string.digits)).strip())","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:35:08.749349Z","iopub.execute_input":"2023-04-10T04:35:08.749706Z","iopub.status.idle":"2023-04-10T04:35:08.835266Z","shell.execute_reply.started":"2023-04-10T04:35:08.749675Z","shell.execute_reply":"2023-04-10T04:35:08.834341Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"##Only use while reading 1 dataset\ntrain_text, temp_text, train_labels, temp_labels = train_test_split(df['text'], df[['Appreciation', 'Complaint', 'Feedback']], \n                                                                    random_state=69, \n                                                                    test_size=0.3, \n                                                                    stratify=df[['Appreciation', 'Complaint', 'Feedback']])\n\n# we will use temp_text and temp_labels to create validation and test set\nval_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels, \n                                                                random_state=69, \n                                                                test_size=0.5, \n                                                                stratify=temp_labels)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:35:08.837473Z","iopub.execute_input":"2023-04-10T04:35:08.837839Z","iopub.status.idle":"2023-04-10T04:35:08.918659Z","shell.execute_reply.started":"2023-04-10T04:35:08.837781Z","shell.execute_reply":"2023-04-10T04:35:08.917522Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"print(train_labels.sum(axis = 0) / train_labels.shape[0])\nprint(val_labels.sum(axis = 0) / val_labels.shape[0])\nprint(test_labels.sum(axis = 0) / test_labels.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:35:08.920106Z","iopub.execute_input":"2023-04-10T04:35:08.920549Z","iopub.status.idle":"2023-04-10T04:35:08.934008Z","shell.execute_reply.started":"2023-04-10T04:35:08.920502Z","shell.execute_reply":"2023-04-10T04:35:08.932580Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Appreciation    0.258973\nComplaint       0.534458\nFeedback        0.206569\ndtype: float64\nAppreciation    0.258794\nComplaint       0.534338\nFeedback        0.206868\ndtype: float64\nAppreciation    0.259414\nComplaint       0.534728\nFeedback        0.205858\ndtype: float64\n","output_type":"stream"}]},{"cell_type":"code","source":"pip install sentence-transformers","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:35:28.867426Z","iopub.execute_input":"2023-04-10T04:35:28.868019Z","iopub.status.idle":"2023-04-10T04:35:41.786790Z","shell.execute_reply.started":"2023-04-10T04:35:28.867984Z","shell.execute_reply":"2023-04-10T04:35:41.785395Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"Collecting sentence-transformers\n  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.6.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (4.27.4)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (4.64.1)\nRequirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.13.0)\nRequirement already satisfied: torchvision in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.14.0)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.21.6)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.0.2)\nRequirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (1.7.3)\nRequirement already satisfied: nltk in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (3.2.4)\nRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.1.97)\nRequirement already satisfied: huggingface-hub>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from sentence-transformers) (0.13.3)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2.28.2)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (6.0)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.11.4)\nRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.9.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.4.0)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.13.2)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2021.11.10)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from nltk->sentence-transformers) (1.16.0)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (3.1.0)\nRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->sentence-transformers) (1.2.0)\nRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.7/site-packages (from torchvision->sentence-transformers) (9.4.0)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->huggingface-hub>=0.4.0->sentence-transformers) (3.11.0)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.4)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2022.12.7)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (1.26.14)\nBuilding wheels for collected packages: sentence-transformers\n  Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125938 sha256=744cc359333b07dc8e6c432e0f4d6317bf30f2311a1aeabe015bfa87bc48bcce\n  Stored in directory: /root/.cache/pip/wheels/83/71/2b/40d17d21937fed496fb99145227eca8f20b4891240ff60c86f\nSuccessfully built sentence-transformers\nInstalling collected packages: sentence-transformers\nSuccessfully installed sentence-transformers-2.2.2\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\nmodel = SentenceTransformer('all-mpnet-base-v2')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:35:55.375120Z","iopub.execute_input":"2023-04-10T04:35:55.376349Z","iopub.status.idle":"2023-04-10T04:36:10.041980Z","shell.execute_reply.started":"2023-04-10T04:35:55.376280Z","shell.execute_reply":"2023-04-10T04:36:10.041056Z"},"trusted":true},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading (…)a8e1d/.gitattributes:   0%|          | 0.00/1.18k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e5bcb09bc8c94364a2021387aa81eda7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)_Pooling/config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a746b4bf857f4c42b83d469e60a11ad2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)b20bca8e1d/README.md:   0%|          | 0.00/10.6k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9126539729634abeacfbaa4957558454"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)0bca8e1d/config.json:   0%|          | 0.00/571 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b54d2083dc8f4b01b5df2a9edaa6b871"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)ce_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"580af8ec025e491cbab8b224ccbcc430"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)e1d/data_config.json:   0%|          | 0.00/39.3k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"51b1f9309351435cb41e52bc6fe46324"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading pytorch_model.bin:   0%|          | 0.00/438M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5bb9e619dada449d9a71e9dfded20979"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)nce_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b2d07e93bb2c40858551fa9bdeda0b9c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)cial_tokens_map.json:   0%|          | 0.00/239 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9f057e5f496243b8808850030601917a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)a8e1d/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b990a02f399140e482d806515c5d855a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)okenizer_config.json:   0%|          | 0.00/363 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b3d8c42f90fc427db7145405fcb4c8eb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)8e1d/train_script.py:   0%|          | 0.00/13.1k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"03883527c1574700bd1f0a1711ec5c84"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)b20bca8e1d/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cf831621d7ba4772a8117d15f736f2c8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)bca8e1d/modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9713bcfa05a54c9d83fafd85172b5726"}},"metadata":{}}]},{"cell_type":"markdown","source":"### Model building","metadata":{}},{"cell_type":"code","source":"embeddings_train = model.encode(train_text.to_list())\nembeddings_val = model.encode(val_text.to_list())\nembeddings_test = model.encode(test_text.to_list())","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:37:02.020880Z","iopub.execute_input":"2023-04-07T05:37:02.021778Z","iopub.status.idle":"2023-04-07T05:37:25.479832Z","shell.execute_reply.started":"2023-04-07T05:37:02.021738Z","shell.execute_reply":"2023-04-07T05:37:25.478639Z"},"trusted":true},"execution_count":10,"outputs":[{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/175 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"93e8656511814904b4ef61b948d2cf2d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/38 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"327c35c29c1b470cae1a33dac8b3a21e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/38 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ed3b9579b0144a5d942a70457141d694"}},"metadata":{}}]},{"cell_type":"code","source":"embeddings_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-07T05:37:38.414284Z","iopub.execute_input":"2023-04-07T05:37:38.414932Z","iopub.status.idle":"2023-04-07T05:37:38.430377Z","shell.execute_reply.started":"2023-04-07T05:37:38.414879Z","shell.execute_reply":"2023-04-07T05:37:38.428966Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"(5572, 768)"},"metadata":{}}]},{"cell_type":"code","source":"f1 = tfa.metrics.F1Score(num_classes = 3, average = 'macro')\n\ncallbacks = [\n    tf.keras.callbacks.EarlyStopping(     #EarlyStopping is used to stop at the epoch where val_accuracy does not improve significantly\n        monitor='val_loss',\n        min_delta=1e-4,\n        patience=5,\n        verbose=1\n    ),\n    tf.keras.callbacks.ModelCheckpoint(\n        filepath='weights_use.h5',\n        monitor='val_loss', \n        mode='min', \n        save_best_only=True,\n        save_weights_only=True,\n        verbose=1\n    )\n]","metadata":{"execution":{"iopub.status.busy":"2023-04-06T21:42:14.096231Z","iopub.execute_input":"2023-04-06T21:42:14.096686Z","iopub.status.idle":"2023-04-06T21:42:14.498218Z","shell.execute_reply.started":"2023-04-06T21:42:14.096651Z","shell.execute_reply":"2023-04-06T21:42:14.497160Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"def SBERT_model_builder():\n    model = tf.keras.models.Sequential()\n    model.add(tf.keras.layers.Input(shape = (768,), dtype = 'float32'))\n    model.add(tf.keras.layers.Dense(256, activation='relu', kernel_regularizer=tf.keras.regularizers.L2(0.01)))\n    #model.add(tf.keras.layers.Dropout(0.1))\n    model.add(tf.keras.layers.Dense(3, activation='softmax'))\n    \n    model.compile(optimizer='adam', \n              loss='categorical_crossentropy', \n              metrics= [f1])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-06T21:43:04.457196Z","iopub.execute_input":"2023-04-06T21:43:04.457650Z","iopub.status.idle":"2023-04-06T21:43:04.465917Z","shell.execute_reply.started":"2023-04-06T21:43:04.457602Z","shell.execute_reply":"2023-04-06T21:43:04.464705Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"sbert_model = SBERT_model_builder()\nsbert_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-06T21:43:15.607613Z","iopub.execute_input":"2023-04-06T21:43:15.608793Z","iopub.status.idle":"2023-04-06T21:43:15.708024Z","shell.execute_reply.started":"2023-04-06T21:43:15.608740Z","shell.execute_reply":"2023-04-06T21:43:15.707211Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stdout","text":"Model: \"sequential\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n dense (Dense)               (None, 256)               196864    \n                                                                 \n dense_1 (Dense)             (None, 3)                 771       \n                                                                 \n=================================================================\nTotal params: 197,635\nTrainable params: 197,635\nNon-trainable params: 0\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"sbert_model.fit(embeddings_train, \n          train_labels, \n          epochs=30, \n          callbacks=callbacks,\n          batch_size = 8,\n          validation_data= (embeddings_val, val_labels))","metadata":{"execution":{"iopub.status.busy":"2023-04-06T21:44:23.051247Z","iopub.execute_input":"2023-04-06T21:44:23.051654Z","iopub.status.idle":"2023-04-06T21:45:41.484034Z","shell.execute_reply.started":"2023-04-06T21:44:23.051615Z","shell.execute_reply":"2023-04-06T21:45:41.482877Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"Epoch 1/30\n693/697 [============================>.] - ETA: 0s - loss: 0.8474 - f1_score: 0.7379\nEpoch 1: val_loss improved from inf to 0.60967, saving model to weights_use.h5\n697/697 [==============================] - 5s 4ms/step - loss: 0.8459 - f1_score: 0.7386 - val_loss: 0.6097 - val_f1_score: 0.8005\nEpoch 2/30\n692/697 [============================>.] - ETA: 0s - loss: 0.6176 - f1_score: 0.7893\nEpoch 2: val_loss improved from 0.60967 to 0.59617, saving model to weights_use.h5\n697/697 [==============================] - 3s 4ms/step - loss: 0.6163 - f1_score: 0.7900 - val_loss: 0.5962 - val_f1_score: 0.8115\nEpoch 3/30\n683/697 [============================>.] - ETA: 0s - loss: 0.6066 - f1_score: 0.7930\nEpoch 3: val_loss did not improve from 0.59617\n697/697 [==============================] - 3s 4ms/step - loss: 0.6077 - f1_score: 0.7918 - val_loss: 0.6295 - val_f1_score: 0.7850\nEpoch 4/30\n693/697 [============================>.] - ETA: 0s - loss: 0.5960 - f1_score: 0.7874\nEpoch 4: val_loss improved from 0.59617 to 0.56367, saving model to weights_use.h5\n697/697 [==============================] - 3s 4ms/step - loss: 0.5958 - f1_score: 0.7876 - val_loss: 0.5637 - val_f1_score: 0.8106\nEpoch 5/30\n695/697 [============================>.] - ETA: 0s - loss: 0.5825 - f1_score: 0.7916\nEpoch 5: val_loss did not improve from 0.56367\n697/697 [==============================] - 2s 3ms/step - loss: 0.5825 - f1_score: 0.7916 - val_loss: 0.5674 - val_f1_score: 0.7942\nEpoch 6/30\n695/697 [============================>.] - ETA: 0s - loss: 0.5785 - f1_score: 0.7962\nEpoch 6: val_loss did not improve from 0.56367\n697/697 [==============================] - 3s 4ms/step - loss: 0.5780 - f1_score: 0.7964 - val_loss: 0.5670 - val_f1_score: 0.7955\nEpoch 7/30\n694/697 [============================>.] - ETA: 0s - loss: 0.5735 - f1_score: 0.7984\nEpoch 7: val_loss did not improve from 0.56367\n697/697 [==============================] - 3s 4ms/step - loss: 0.5741 - f1_score: 0.7981 - val_loss: 0.5785 - val_f1_score: 0.7984\nEpoch 8/30\n693/697 [============================>.] - ETA: 0s - loss: 0.5750 - f1_score: 0.8014\nEpoch 8: val_loss did not improve from 0.56367\n697/697 [==============================] - 3s 4ms/step - loss: 0.5748 - f1_score: 0.8021 - val_loss: 0.5840 - val_f1_score: 0.8189\nEpoch 9/30\n695/697 [============================>.] - ETA: 0s - loss: 0.5668 - f1_score: 0.7982\nEpoch 9: val_loss improved from 0.56367 to 0.54039, saving model to weights_use.h5\n697/697 [==============================] - 3s 4ms/step - loss: 0.5664 - f1_score: 0.7985 - val_loss: 0.5404 - val_f1_score: 0.8288\nEpoch 10/30\n694/697 [============================>.] - ETA: 0s - loss: 0.5655 - f1_score: 0.7996\nEpoch 10: val_loss did not improve from 0.54039\n697/697 [==============================] - 3s 4ms/step - loss: 0.5647 - f1_score: 0.8001 - val_loss: 0.5991 - val_f1_score: 0.7917\nEpoch 11/30\n694/697 [============================>.] - ETA: 0s - loss: 0.5612 - f1_score: 0.8017\nEpoch 11: val_loss improved from 0.54039 to 0.53398, saving model to weights_use.h5\n697/697 [==============================] - 3s 4ms/step - loss: 0.5610 - f1_score: 0.8016 - val_loss: 0.5340 - val_f1_score: 0.8181\nEpoch 12/30\n684/697 [============================>.] - ETA: 0s - loss: 0.5590 - f1_score: 0.7985\nEpoch 12: val_loss did not improve from 0.53398\n697/697 [==============================] - 2s 4ms/step - loss: 0.5577 - f1_score: 0.8001 - val_loss: 0.5725 - val_f1_score: 0.7965\nEpoch 13/30\n691/697 [============================>.] - ETA: 0s - loss: 0.5520 - f1_score: 0.8036\nEpoch 13: val_loss did not improve from 0.53398\n697/697 [==============================] - 2s 3ms/step - loss: 0.5516 - f1_score: 0.8038 - val_loss: 0.5589 - val_f1_score: 0.7968\nEpoch 14/30\n689/697 [============================>.] - ETA: 0s - loss: 0.5404 - f1_score: 0.8097\nEpoch 14: val_loss improved from 0.53398 to 0.53319, saving model to weights_use.h5\n697/697 [==============================] - 2s 3ms/step - loss: 0.5419 - f1_score: 0.8090 - val_loss: 0.5332 - val_f1_score: 0.8018\nEpoch 15/30\n683/697 [============================>.] - ETA: 0s - loss: 0.5502 - f1_score: 0.8048\nEpoch 15: val_loss did not improve from 0.53319\n697/697 [==============================] - 2s 3ms/step - loss: 0.5491 - f1_score: 0.8050 - val_loss: 0.5752 - val_f1_score: 0.7834\nEpoch 16/30\n686/697 [============================>.] - ETA: 0s - loss: 0.5344 - f1_score: 0.8051\nEpoch 16: val_loss improved from 0.53319 to 0.52504, saving model to weights_use.h5\n697/697 [==============================] - 3s 4ms/step - loss: 0.5349 - f1_score: 0.8057 - val_loss: 0.5250 - val_f1_score: 0.8192\nEpoch 17/30\n685/697 [============================>.] - ETA: 0s - loss: 0.5277 - f1_score: 0.8157\nEpoch 17: val_loss improved from 0.52504 to 0.52397, saving model to weights_use.h5\n697/697 [==============================] - 3s 4ms/step - loss: 0.5264 - f1_score: 0.8154 - val_loss: 0.5240 - val_f1_score: 0.8254\nEpoch 18/30\n691/697 [============================>.] - ETA: 0s - loss: 0.5351 - f1_score: 0.8067\nEpoch 18: val_loss did not improve from 0.52397\n697/697 [==============================] - 2s 3ms/step - loss: 0.5338 - f1_score: 0.8070 - val_loss: 0.5416 - val_f1_score: 0.8028\nEpoch 19/30\n680/697 [============================>.] - ETA: 0s - loss: 0.5258 - f1_score: 0.8145\nEpoch 19: val_loss did not improve from 0.52397\n697/697 [==============================] - 2s 3ms/step - loss: 0.5257 - f1_score: 0.8132 - val_loss: 0.5249 - val_f1_score: 0.8151\nEpoch 20/30\n680/697 [============================>.] - ETA: 0s - loss: 0.5223 - f1_score: 0.8140\nEpoch 20: val_loss improved from 0.52397 to 0.52257, saving model to weights_use.h5\n697/697 [==============================] - 2s 3ms/step - loss: 0.5226 - f1_score: 0.8142 - val_loss: 0.5226 - val_f1_score: 0.8043\nEpoch 21/30\n693/697 [============================>.] - ETA: 0s - loss: 0.5208 - f1_score: 0.8165\nEpoch 21: val_loss did not improve from 0.52257\n697/697 [==============================] - 2s 4ms/step - loss: 0.5201 - f1_score: 0.8164 - val_loss: 0.5320 - val_f1_score: 0.7999\nEpoch 22/30\n681/697 [============================>.] - ETA: 0s - loss: 0.5150 - f1_score: 0.8130\nEpoch 22: val_loss did not improve from 0.52257\n697/697 [==============================] - 3s 4ms/step - loss: 0.5176 - f1_score: 0.8111 - val_loss: 0.5462 - val_f1_score: 0.8159\nEpoch 23/30\n682/697 [============================>.] - ETA: 0s - loss: 0.5163 - f1_score: 0.8154\nEpoch 23: val_loss did not improve from 0.52257\n697/697 [==============================] - 3s 4ms/step - loss: 0.5160 - f1_score: 0.8154 - val_loss: 0.5416 - val_f1_score: 0.8082\nEpoch 24/30\n687/697 [============================>.] - ETA: 0s - loss: 0.5079 - f1_score: 0.8183\nEpoch 24: val_loss improved from 0.52257 to 0.50432, saving model to weights_use.h5\n697/697 [==============================] - 2s 3ms/step - loss: 0.5064 - f1_score: 0.8189 - val_loss: 0.5043 - val_f1_score: 0.8200\nEpoch 25/30\n687/697 [============================>.] - ETA: 0s - loss: 0.5091 - f1_score: 0.8159\nEpoch 25: val_loss did not improve from 0.50432\n697/697 [==============================] - 2s 4ms/step - loss: 0.5078 - f1_score: 0.8164 - val_loss: 0.5214 - val_f1_score: 0.8078\nEpoch 26/30\n694/697 [============================>.] - ETA: 0s - loss: 0.5065 - f1_score: 0.8154\nEpoch 26: val_loss did not improve from 0.50432\n697/697 [==============================] - 2s 3ms/step - loss: 0.5061 - f1_score: 0.8157 - val_loss: 0.5064 - val_f1_score: 0.8267\nEpoch 27/30\n691/697 [============================>.] - ETA: 0s - loss: 0.5073 - f1_score: 0.8146\nEpoch 27: val_loss did not improve from 0.50432\n697/697 [==============================] - 2s 3ms/step - loss: 0.5068 - f1_score: 0.8147 - val_loss: 0.5397 - val_f1_score: 0.7992\nEpoch 28/30\n696/697 [============================>.] - ETA: 0s - loss: 0.5046 - f1_score: 0.8132\nEpoch 28: val_loss improved from 0.50432 to 0.49509, saving model to weights_use.h5\n697/697 [==============================] - 2s 3ms/step - loss: 0.5043 - f1_score: 0.8133 - val_loss: 0.4951 - val_f1_score: 0.8172\nEpoch 29/30\n693/697 [============================>.] - ETA: 0s - loss: 0.4977 - f1_score: 0.8215\nEpoch 29: val_loss did not improve from 0.49509\n697/697 [==============================] - 2s 3ms/step - loss: 0.4972 - f1_score: 0.8216 - val_loss: 0.5060 - val_f1_score: 0.8042\nEpoch 30/30\n694/697 [============================>.] - ETA: 0s - loss: 0.4970 - f1_score: 0.8201\nEpoch 30: val_loss did not improve from 0.49509\n697/697 [==============================] - 2s 4ms/step - loss: 0.4974 - f1_score: 0.8198 - val_loss: 0.5098 - val_f1_score: 0.8173\n","output_type":"stream"},{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7985f2dd9290>"},"metadata":{}}]},{"cell_type":"code","source":"sbert_model.evaluate(embeddings_test, test_labels)","metadata":{"execution":{"iopub.status.busy":"2023-04-06T21:46:23.206829Z","iopub.execute_input":"2023-04-06T21:46:23.207646Z","iopub.status.idle":"2023-04-06T21:46:23.412555Z","shell.execute_reply.started":"2023-04-06T21:46:23.207575Z","shell.execute_reply":"2023-04-06T21:46:23.411577Z"},"trusted":true},"execution_count":24,"outputs":[{"name":"stdout","text":"38/38 [==============================] - 0s 3ms/step - loss: 0.5220 - f1_score: 0.7982\n","output_type":"stream"},{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"[0.5219564437866211, 0.7982286810874939]"},"metadata":{}}]},{"cell_type":"markdown","source":"### HP tuning","metadata":{}},{"cell_type":"code","source":"pip install -q -U keras-tuner","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:09:37.007157Z","iopub.execute_input":"2023-04-06T22:09:37.010906Z","iopub.status.idle":"2023-04-06T22:10:01.885316Z","shell.execute_reply.started":"2023-04-06T22:09:37.010865Z","shell.execute_reply":"2023-04-06T22:10:01.883874Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\nTo disable this warning, you can either:\n\t- Avoid using `tokenizers` before the fork if possible\n\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ncudf 21.12.2 requires cupy-cuda115, which is not installed.\ntfx-bsl 1.12.0 requires google-api-python-client<2,>=1.7.11, but you have google-api-python-client 2.83.0 which is incompatible.\ntfx-bsl 1.12.0 requires pyarrow<7,>=6, but you have pyarrow 5.0.0 which is incompatible.\ntensorflow-transform 1.12.0 requires pyarrow<7,>=6, but you have pyarrow 5.0.0 which is incompatible.\nonnx 1.13.1 requires protobuf<4,>=3.20.2, but you have protobuf 3.19.6 which is incompatible.\napache-beam 2.44.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.6 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"import keras_tuner as kt","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:10:01.887517Z","iopub.execute_input":"2023-04-06T22:10:01.887923Z","iopub.status.idle":"2023-04-06T22:10:01.963061Z","shell.execute_reply.started":"2023-04-06T22:10:01.887874Z","shell.execute_reply":"2023-04-06T22:10:01.961880Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"f1 = tfa.metrics.F1Score(num_classes = 3, average = 'macro')\n\ndef sbert_model_builder_HP(hp):\n    model = tf.keras.models.Sequential()\n    \n    #Input layer\n    model.add(tf.keras.layers.Input(shape = (768,), dtype = 'float32'))\n    \n    #Layer 1\n    hp_units = hp.Int('units', min_value=192, max_value=512, step=32)\n    hp_l2 = hp.Choice('l2', values=[1e-1, 1e-2, 1e-3, 1e-4])\n    model.add(tf.keras.layers.Dense(units = hp_units, activation='relu', kernel_regularizer=tf.keras.regularizers.L2(hp_l2)))\n    \n    #Layer 2\n    model.add(tf.keras.layers.Dense(units = hp_units, activation='relu'))\n    \n    #Layer 3\n    hp_dropout = hp.Choice('Dropout', values=[0.0, 0.05,0.1])\n    model.add(tf.keras.layers.Dropout(hp_dropout))\n    \n    # Output layer\n    model.add(tf.keras.layers.Dense(3, activation='softmax'))\n    \n    hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4]) \n    model.compile(optimizer= tf.keras.optimizers.Adam(learning_rate=hp_learning_rate),\n                loss='categorical_crossentropy',\n                metrics=[f1])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:16:02.392289Z","iopub.execute_input":"2023-04-06T22:16:02.393344Z","iopub.status.idle":"2023-04-06T22:16:02.413960Z","shell.execute_reply.started":"2023-04-06T22:16:02.393303Z","shell.execute_reply":"2023-04-06T22:16:02.412553Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"tuner = kt.Hyperband(sbert_model_builder_HP,\n                     objective= kt.Objective(\"val_f1_score\", direction=\"max\"),\n                     max_epochs=50,\n                     factor=3)\n\nstop_early = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience=5)\n\ntuner.search(embeddings_train, train_labels,\n             epochs= 25,\n             validation_data=(embeddings_val, val_labels), \n             callbacks=[stop_early])","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:16:21.776662Z","iopub.execute_input":"2023-04-06T22:16:21.777234Z","iopub.status.idle":"2023-04-06T22:26:47.500677Z","shell.execute_reply.started":"2023-04-06T22:16:21.777195Z","shell.execute_reply":"2023-04-06T22:26:47.499705Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"Trial 90 Complete [00h 00m 12s]\nval_f1_score: 0.8320834040641785\n\nBest val_f1_score So Far: 0.8534145355224609\nTotal elapsed time: 00h 10m 26s\n","output_type":"stream"}]},{"cell_type":"code","source":"best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]\n\nsbert_tuned_model = tuner.hypermodel.build(best_hps)\nhistory = sbert_tuned_model.fit(embeddings_train, train_labels, epochs=50, validation_data=(embeddings_val, val_labels))\n\nval_f1_per_epoch = history.history['val_f1_score']\nbest_epoch = val_f1_per_epoch.index(max(val_f1_per_epoch)) + 1\nprint('Best epoch: %d' % (best_epoch,))","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:26:47.502874Z","iopub.execute_input":"2023-04-06T22:26:47.503445Z","iopub.status.idle":"2023-04-06T22:27:35.452775Z","shell.execute_reply.started":"2023-04-06T22:26:47.503403Z","shell.execute_reply":"2023-04-06T22:27:35.451693Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"Epoch 1/50\n175/175 [==============================] - 2s 5ms/step - loss: 0.5235 - f1_score: 0.7899 - val_loss: 0.4340 - val_f1_score: 0.8338\nEpoch 2/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.3879 - f1_score: 0.8545 - val_loss: 0.4424 - val_f1_score: 0.8368\nEpoch 3/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.3300 - f1_score: 0.8860 - val_loss: 0.4340 - val_f1_score: 0.8509\nEpoch 4/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.2697 - f1_score: 0.9116 - val_loss: 0.4653 - val_f1_score: 0.8439\nEpoch 5/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.2151 - f1_score: 0.9391 - val_loss: 0.5104 - val_f1_score: 0.8452\nEpoch 6/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.1600 - f1_score: 0.9653 - val_loss: 0.6048 - val_f1_score: 0.8404\nEpoch 7/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.1301 - f1_score: 0.9773 - val_loss: 0.6521 - val_f1_score: 0.8257\nEpoch 8/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.1108 - f1_score: 0.9847 - val_loss: 0.7005 - val_f1_score: 0.8171\nEpoch 9/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.1007 - f1_score: 0.9880 - val_loss: 0.7245 - val_f1_score: 0.8339\nEpoch 10/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0926 - f1_score: 0.9896 - val_loss: 0.8300 - val_f1_score: 0.8189\nEpoch 11/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0896 - f1_score: 0.9917 - val_loss: 0.7672 - val_f1_score: 0.8390\nEpoch 12/50\n175/175 [==============================] - 1s 5ms/step - loss: 0.1022 - f1_score: 0.9876 - val_loss: 0.7625 - val_f1_score: 0.8399\nEpoch 13/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0847 - f1_score: 0.9925 - val_loss: 0.7723 - val_f1_score: 0.8354\nEpoch 14/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0922 - f1_score: 0.9897 - val_loss: 0.8798 - val_f1_score: 0.8237\nEpoch 15/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0807 - f1_score: 0.9939 - val_loss: 0.8841 - val_f1_score: 0.8296\nEpoch 16/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0836 - f1_score: 0.9918 - val_loss: 0.9505 - val_f1_score: 0.8149\nEpoch 17/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0905 - f1_score: 0.9889 - val_loss: 0.9011 - val_f1_score: 0.8139\nEpoch 18/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0869 - f1_score: 0.9901 - val_loss: 0.9804 - val_f1_score: 0.8241\nEpoch 19/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0932 - f1_score: 0.9883 - val_loss: 0.9589 - val_f1_score: 0.8223\nEpoch 20/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0858 - f1_score: 0.9917 - val_loss: 0.9234 - val_f1_score: 0.8237\nEpoch 21/50\n175/175 [==============================] - 1s 5ms/step - loss: 0.0685 - f1_score: 0.9974 - val_loss: 0.9098 - val_f1_score: 0.8238\nEpoch 22/50\n175/175 [==============================] - 1s 7ms/step - loss: 0.0601 - f1_score: 0.9992 - val_loss: 0.8961 - val_f1_score: 0.8393\nEpoch 23/50\n175/175 [==============================] - 1s 6ms/step - loss: 0.0552 - f1_score: 0.9994 - val_loss: 0.8923 - val_f1_score: 0.8360\nEpoch 24/50\n175/175 [==============================] - 1s 5ms/step - loss: 0.0511 - f1_score: 0.9990 - val_loss: 0.8852 - val_f1_score: 0.8444\nEpoch 25/50\n175/175 [==============================] - 1s 5ms/step - loss: 0.0468 - f1_score: 0.9992 - val_loss: 0.8734 - val_f1_score: 0.8379\nEpoch 26/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0436 - f1_score: 0.9992 - val_loss: 0.8607 - val_f1_score: 0.8415\nEpoch 27/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0392 - f1_score: 0.9994 - val_loss: 0.8533 - val_f1_score: 0.8423\nEpoch 28/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0359 - f1_score: 0.9992 - val_loss: 0.8394 - val_f1_score: 0.8415\nEpoch 29/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0325 - f1_score: 0.9992 - val_loss: 0.8243 - val_f1_score: 0.8439\nEpoch 30/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0294 - f1_score: 0.9994 - val_loss: 0.8134 - val_f1_score: 0.8412\nEpoch 31/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0265 - f1_score: 0.9992 - val_loss: 0.8016 - val_f1_score: 0.8434\nEpoch 32/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.1204 - f1_score: 0.9705 - val_loss: 0.6219 - val_f1_score: 0.8100\nEpoch 33/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.1941 - f1_score: 0.9341 - val_loss: 0.5775 - val_f1_score: 0.8252\nEpoch 34/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0848 - f1_score: 0.9841 - val_loss: 0.7497 - val_f1_score: 0.8409\nEpoch 35/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0661 - f1_score: 0.9934 - val_loss: 0.8046 - val_f1_score: 0.8326\nEpoch 36/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0551 - f1_score: 0.9957 - val_loss: 0.8415 - val_f1_score: 0.8302\nEpoch 37/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0671 - f1_score: 0.9906 - val_loss: 0.8942 - val_f1_score: 0.8150\nEpoch 38/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0506 - f1_score: 0.9973 - val_loss: 0.9055 - val_f1_score: 0.8307\nEpoch 39/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0537 - f1_score: 0.9968 - val_loss: 0.9085 - val_f1_score: 0.8392\nEpoch 40/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0412 - f1_score: 0.9992 - val_loss: 0.9229 - val_f1_score: 0.8425\nEpoch 41/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0383 - f1_score: 0.9992 - val_loss: 0.9161 - val_f1_score: 0.8376\nEpoch 42/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0360 - f1_score: 0.9994 - val_loss: 0.9139 - val_f1_score: 0.8381\nEpoch 43/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0337 - f1_score: 0.9994 - val_loss: 0.9056 - val_f1_score: 0.8392\nEpoch 44/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0314 - f1_score: 0.9994 - val_loss: 0.9043 - val_f1_score: 0.8440\nEpoch 45/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0298 - f1_score: 0.9992 - val_loss: 0.8882 - val_f1_score: 0.8375\nEpoch 46/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0275 - f1_score: 0.9994 - val_loss: 0.9144 - val_f1_score: 0.8384\nEpoch 47/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0266 - f1_score: 0.9990 - val_loss: 0.8816 - val_f1_score: 0.8406\nEpoch 48/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0242 - f1_score: 0.9992 - val_loss: 0.8892 - val_f1_score: 0.8394\nEpoch 49/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0223 - f1_score: 0.9992 - val_loss: 0.8591 - val_f1_score: 0.8404\nEpoch 50/50\n175/175 [==============================] - 1s 4ms/step - loss: 0.0205 - f1_score: 0.9994 - val_loss: 0.8654 - val_f1_score: 0.8443\nBest epoch: 3\n","output_type":"stream"}]},{"cell_type":"code","source":"hypermodel = tuner.hypermodel.build(best_hps)\n\n# Retrain the model\nhypermodel.fit(embeddings_train, train_labels, epochs=best_epoch, validation_data=(embeddings_val, val_labels))","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:27:35.454562Z","iopub.execute_input":"2023-04-06T22:27:35.454952Z","iopub.status.idle":"2023-04-06T22:27:38.851953Z","shell.execute_reply.started":"2023-04-06T22:27:35.454905Z","shell.execute_reply":"2023-04-06T22:27:38.850880Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"Epoch 1/3\n175/175 [==============================] - 2s 5ms/step - loss: 0.5311 - f1_score: 0.7923 - val_loss: 0.4344 - val_f1_score: 0.8404\nEpoch 2/3\n175/175 [==============================] - 1s 4ms/step - loss: 0.3837 - f1_score: 0.8554 - val_loss: 0.4337 - val_f1_score: 0.8377\nEpoch 3/3\n175/175 [==============================] - 1s 4ms/step - loss: 0.3246 - f1_score: 0.8869 - val_loss: 0.4316 - val_f1_score: 0.8389\n","output_type":"stream"},{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"<keras.callbacks.History at 0x7dc550526090>"},"metadata":{}}]},{"cell_type":"code","source":"eval_result = hypermodel.evaluate(embeddings_test, test_labels)\nprint(\"[test loss, test f1]:\", eval_result)","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:27:38.857569Z","iopub.execute_input":"2023-04-06T22:27:38.860011Z","iopub.status.idle":"2023-04-06T22:27:39.036282Z","shell.execute_reply.started":"2023-04-06T22:27:38.859970Z","shell.execute_reply":"2023-04-06T22:27:39.035158Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"38/38 [==============================] - 0s 3ms/step - loss: 0.4227 - f1_score: 0.8370\n[test loss, test f1]: [0.42269405722618103, 0.8369593620300293]\n","output_type":"stream"}]},{"cell_type":"code","source":"hypermodel.save('fb_hyper_sbert.hdf5')","metadata":{"execution":{"iopub.status.busy":"2023-04-06T22:27:39.037699Z","iopub.execute_input":"2023-04-06T22:27:39.038219Z","iopub.status.idle":"2023-04-06T22:27:39.080944Z","shell.execute_reply.started":"2023-04-06T22:27:39.038179Z","shell.execute_reply":"2023-04-06T22:27:39.079947Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"markdown","source":"### Stacking model","metadata":{}},{"cell_type":"code","source":"f1 = tfa.metrics.F1Score(num_classes = 3, average = 'macro')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:36:39.760466Z","iopub.execute_input":"2023-04-10T04:36:39.761647Z","iopub.status.idle":"2023-04-10T04:36:42.318248Z","shell.execute_reply.started":"2023-04-10T04:36:39.761595Z","shell.execute_reply":"2023-04-10T04:36:42.317192Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"embeddings_val = model.encode(val_text.to_list())\nembeddings_test = model.encode(test_text.to_list())","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:36:18.225578Z","iopub.execute_input":"2023-04-10T04:36:18.225977Z","iopub.status.idle":"2023-04-10T04:36:34.205575Z","shell.execute_reply.started":"2023-04-10T04:36:18.225933Z","shell.execute_reply":"2023-04-10T04:36:34.204512Z"},"trusted":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/38 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ae99f5af61eb41a9b57ec4cec7502434"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/38 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5e9a548b538d4ac2b53cc021ca4a3091"}},"metadata":{}}]},{"cell_type":"code","source":"print(embeddings_val.shape)\nprint(embeddings_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:14:54.416440Z","iopub.execute_input":"2023-04-10T05:14:54.417254Z","iopub.status.idle":"2023-04-10T05:14:54.424979Z","shell.execute_reply.started":"2023-04-10T05:14:54.417213Z","shell.execute_reply":"2023-04-10T05:14:54.423374Z"},"trusted":true},"execution_count":75,"outputs":[{"name":"stdout","text":"(1194, 768)\n(1195, 768)\n","output_type":"stream"}]},{"cell_type":"code","source":"all_embeds = np.append(embeddings_val, embeddings_test, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:15:48.969397Z","iopub.execute_input":"2023-04-10T05:15:48.970139Z","iopub.status.idle":"2023-04-10T05:15:48.976705Z","shell.execute_reply.started":"2023-04-10T05:15:48.970101Z","shell.execute_reply":"2023-04-10T05:15:48.975539Z"},"trusted":true},"execution_count":80,"outputs":[]},{"cell_type":"code","source":"all_embeds.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:15:53.877400Z","iopub.execute_input":"2023-04-10T05:15:53.877924Z","iopub.status.idle":"2023-04-10T05:15:53.886318Z","shell.execute_reply.started":"2023-04-10T05:15:53.877872Z","shell.execute_reply":"2023-04-10T05:15:53.884887Z"},"trusted":true},"execution_count":81,"outputs":[{"execution_count":81,"output_type":"execute_result","data":{"text/plain":"(2389, 768)"},"metadata":{}}]},{"cell_type":"code","source":"all_labels = pd.concat([val_labels, test_labels], axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:18:06.945746Z","iopub.execute_input":"2023-04-10T05:18:06.946365Z","iopub.status.idle":"2023-04-10T05:18:06.953471Z","shell.execute_reply.started":"2023-04-10T05:18:06.946325Z","shell.execute_reply":"2023-04-10T05:18:06.951288Z"},"trusted":true},"execution_count":93,"outputs":[]},{"cell_type":"code","source":"inputy = all_labels.idxmax(1)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:18:11.702314Z","iopub.execute_input":"2023-04-10T05:18:11.703037Z","iopub.status.idle":"2023-04-10T05:18:11.713528Z","shell.execute_reply.started":"2023-04-10T05:18:11.702997Z","shell.execute_reply":"2023-04-10T05:18:11.712441Z"},"trusted":true},"execution_count":95,"outputs":[]},{"cell_type":"code","source":"all_text = pd.concat([val_text, test_text], axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:20:37.313397Z","iopub.execute_input":"2023-04-10T05:20:37.314455Z","iopub.status.idle":"2023-04-10T05:20:37.320642Z","shell.execute_reply.started":"2023-04-10T05:20:37.314404Z","shell.execute_reply":"2023-04-10T05:20:37.319345Z"},"trusted":true},"execution_count":102,"outputs":[]},{"cell_type":"code","source":"all_text.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:20:44.528594Z","iopub.execute_input":"2023-04-10T05:20:44.529750Z","iopub.status.idle":"2023-04-10T05:20:44.538130Z","shell.execute_reply.started":"2023-04-10T05:20:44.529706Z","shell.execute_reply":"2023-04-10T05:20:44.536933Z"},"trusted":true},"execution_count":104,"outputs":[{"execution_count":104,"output_type":"execute_result","data":{"text/plain":"(2389,)"},"metadata":{}}]},{"cell_type":"code","source":"KerasLayer = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder-large/5', \n                        input_shape=[], \n                        dtype=tf.string, \n                        trainable=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:37:41.052305Z","iopub.execute_input":"2023-04-10T04:37:41.053060Z","iopub.status.idle":"2023-04-10T04:38:04.105127Z","shell.execute_reply.started":"2023-04-10T04:37:41.053021Z","shell.execute_reply":"2023-04-10T04:38:04.103967Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"# load models from file\ndef load_all_models():\n    all_models = list()\n    model_names = ['/kaggle/input/models-for-stacking/fb_hyper_sbert.hdf5',\n                   '/kaggle/input/models-for-stacking/fb_hyper_use_LARGE.hdf5',\n                   '/kaggle/input/models-for-stacking/fb_hyper_use_large_class.hdf5']\n    for i in range(len(model_names)):\n        # load models\n        model = tf.keras.models.load_model(model_names[i],custom_objects={'KerasLayer': KerasLayer,\n                                                          'f1' : f1})\n        # add to list of members\n        all_models.append(model)\n        print('>loaded %s' % model_names[i])\n    return all_models","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:18:25.325241Z","iopub.execute_input":"2023-04-10T05:18:25.325864Z","iopub.status.idle":"2023-04-10T05:18:25.332213Z","shell.execute_reply.started":"2023-04-10T05:18:25.325825Z","shell.execute_reply":"2023-04-10T05:18:25.330963Z"},"trusted":true},"execution_count":97,"outputs":[]},{"cell_type":"code","source":"members = load_all_models()\nprint('Loaded %d models' % len(members))","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:18:29.831350Z","iopub.execute_input":"2023-04-10T05:18:29.832114Z","iopub.status.idle":"2023-04-10T05:19:13.538370Z","shell.execute_reply.started":"2023-04-10T05:18:29.832074Z","shell.execute_reply":"2023-04-10T05:19:13.537260Z"},"trusted":true},"execution_count":98,"outputs":[{"name":"stdout","text":">loaded /kaggle/input/models-for-stacking/fb_hyper_sbert.hdf5\n>loaded /kaggle/input/models-for-stacking/fb_hyper_use_LARGE.hdf5\n>loaded /kaggle/input/models-for-stacking/fb_hyper_use_large_class.hdf5\nLoaded 3 models\n","output_type":"stream"}]},{"cell_type":"code","source":"members[0].summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:39:13.701494Z","iopub.execute_input":"2023-04-10T04:39:13.702363Z","iopub.status.idle":"2023-04-10T04:39:13.728182Z","shell.execute_reply.started":"2023-04-10T04:39:13.702327Z","shell.execute_reply":"2023-04-10T04:39:13.727301Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"Model: \"sequential_2\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n dense_6 (Dense)             (None, 448)               344512    \n                                                                 \n dense_7 (Dense)             (None, 448)               201152    \n                                                                 \n dropout_2 (Dropout)         (None, 448)               0         \n                                                                 \n dense_8 (Dense)             (None, 3)                 1347      \n                                                                 \n=================================================================\nTotal params: 547,011\nTrainable params: 547,011\nNon-trainable params: 0\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"members[1].summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:39:16.168289Z","iopub.execute_input":"2023-04-10T04:39:16.168884Z","iopub.status.idle":"2023-04-10T04:39:16.251643Z","shell.execute_reply.started":"2023-04-10T04:39:16.168845Z","shell.execute_reply":"2023-04-10T04:39:16.250409Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"Model: \"sequential_2\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n keras_layer_2 (KerasLayer)  (None, 512)               147354880 \n                                                                 \n dense_6 (Dense)             (None, 384)               196992    \n                                                                 \n dense_7 (Dense)             (None, 384)               147840    \n                                                                 \n dropout_2 (Dropout)         (None, 384)               0         \n                                                                 \n dense_8 (Dense)             (None, 3)                 1155      \n                                                                 \n=================================================================\nTotal params: 147,700,867\nTrainable params: 345,987\nNon-trainable params: 147,354,880\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"members[2].summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:39:18.539004Z","iopub.execute_input":"2023-04-10T04:39:18.540295Z","iopub.status.idle":"2023-04-10T04:39:18.623873Z","shell.execute_reply.started":"2023-04-10T04:39:18.540249Z","shell.execute_reply":"2023-04-10T04:39:18.622783Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"Model: \"sequential_2\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n keras_layer_2 (KerasLayer)  (None, 512)               147354880 \n                                                                 \n dense_6 (Dense)             (None, 448)               229824    \n                                                                 \n dense_7 (Dense)             (None, 448)               201152    \n                                                                 \n dropout_2 (Dropout)         (None, 448)               0         \n                                                                 \n dense_8 (Dense)             (None, 3)                 1347      \n                                                                 \n=================================================================\nTotal params: 147,787,203\nTrainable params: 432,323\nNon-trainable params: 147,354,880\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"# create stacked model input dataset as outputs from the ensemble\ndef stacked_dataset(members, inputX_sbert, inputX_use):\n    stackX = None\n    # make prediction\n    yhat0 = members[0].predict(inputX_sbert, verbose=0)\n    # stack predictions into [rows, members, probabilities]\n    stackX = yhat0 #\n\n    yhat1 = members[1].predict(inputX_use, verbose=0)\n    stackX = dstack((stackX, yhat1))\n    \n    yhat2 = members[2].predict(inputX_use, verbose=0)\n    stackX = dstack((stackX, yhat2))\n    \n    # flatten predictions to [rows, members x probabilities]\n    stackX = stackX.reshape((stackX.shape[0], stackX.shape[1]*stackX.shape[2]))\n    return stackX","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:19:13.541513Z","iopub.execute_input":"2023-04-10T05:19:13.543517Z","iopub.status.idle":"2023-04-10T05:19:13.550912Z","shell.execute_reply.started":"2023-04-10T05:19:13.543476Z","shell.execute_reply":"2023-04-10T05:19:13.549893Z"},"trusted":true},"execution_count":99,"outputs":[]},{"cell_type":"code","source":"test_stack = stacked_dataset(members, all_embeds, all_text)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:21:01.338576Z","iopub.execute_input":"2023-04-10T05:21:01.339547Z","iopub.status.idle":"2023-04-10T05:21:34.369700Z","shell.execute_reply.started":"2023-04-10T05:21:01.339491Z","shell.execute_reply":"2023-04-10T05:21:34.368579Z"},"trusted":true},"execution_count":105,"outputs":[]},{"cell_type":"code","source":"test_stack.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:21:34.371798Z","iopub.execute_input":"2023-04-10T05:21:34.372474Z","iopub.status.idle":"2023-04-10T05:21:34.379876Z","shell.execute_reply.started":"2023-04-10T05:21:34.372419Z","shell.execute_reply":"2023-04-10T05:21:34.378772Z"},"trusted":true},"execution_count":106,"outputs":[{"execution_count":106,"output_type":"execute_result","data":{"text/plain":"(2389, 9)"},"metadata":{}}]},{"cell_type":"code","source":"labels = {'Appreciation': 0,\n             'Complaint': 1, \n             'Feedback': 2}\n\ninputy = inputy.map(labels) # re-mapping","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:21:56.192299Z","iopub.execute_input":"2023-04-10T05:21:56.193018Z","iopub.status.idle":"2023-04-10T05:21:56.199208Z","shell.execute_reply.started":"2023-04-10T05:21:56.192979Z","shell.execute_reply":"2023-04-10T05:21:56.197982Z"},"trusted":true},"execution_count":108,"outputs":[]},{"cell_type":"code","source":"inputy","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:21:59.513864Z","iopub.execute_input":"2023-04-10T05:21:59.514342Z","iopub.status.idle":"2023-04-10T05:21:59.527161Z","shell.execute_reply.started":"2023-04-10T05:21:59.514305Z","shell.execute_reply":"2023-04-10T05:21:59.525813Z"},"trusted":true},"execution_count":109,"outputs":[{"execution_count":109,"output_type":"execute_result","data":{"text/plain":"5905    1\n1839    1\n6975    1\n865     0\n4194    0\n       ..\n1235    2\n4336    2\n6882    0\n6418    2\n7526    1\nLength: 2389, dtype: int64"},"metadata":{}}]},{"cell_type":"code","source":"#import xgboost as xgb\n\n# fit a model based on the outputs from the ensemble members\ndef fit_stacked_model(members, inputX_sbert, inputX_use, inputy):\n    # create dataset using ensemble\n    stackedX = stacked_dataset(members, inputX_sbert, inputX_use)\n    # fit standalone model\n#     model = xgb.XGBClassifier(objective='multi:softmax', \n#                             num_class=3) #meta learner\n    model = LogisticRegression(multi_class = \"ovr\", class_weight ={0: 1, 1 : 1, 2: 3})\n    model.fit(stackedX, inputy)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:22:39.495216Z","iopub.execute_input":"2023-04-10T05:22:39.495773Z","iopub.status.idle":"2023-04-10T05:22:39.502669Z","shell.execute_reply.started":"2023-04-10T05:22:39.495735Z","shell.execute_reply":"2023-04-10T05:22:39.501503Z"},"trusted":true},"execution_count":110,"outputs":[]},{"cell_type":"code","source":"stacked_model = fit_stacked_model(members, all_embeds, all_text, inputy)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:23:08.586834Z","iopub.execute_input":"2023-04-10T05:23:08.587214Z","iopub.status.idle":"2023-04-10T05:23:29.632503Z","shell.execute_reply.started":"2023-04-10T05:23:08.587181Z","shell.execute_reply":"2023-04-10T05:23:29.631012Z"},"trusted":true},"execution_count":112,"outputs":[]},{"cell_type":"code","source":"import pickle\nfilename = 'stacked_model_log_v3.sav'\npickle.dump(stacked_model, open(filename, 'wb'))\n#stacked_model = pickle.load(open(filename, 'rb'))","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:23:29.640065Z","iopub.execute_input":"2023-04-10T05:23:29.643564Z","iopub.status.idle":"2023-04-10T05:23:29.652615Z","shell.execute_reply.started":"2023-04-10T05:23:29.643506Z","shell.execute_reply":"2023-04-10T05:23:29.651249Z"},"trusted":true},"execution_count":113,"outputs":[]},{"cell_type":"code","source":"# make a prediction with the stacked model\ndef stacked_prediction(members, model, inputX_sbert, inputX_use):\n    # create dataset using ensemble\n    stackedX = stacked_dataset(members, inputX_sbert, inputX_use)\n    # make a prediction\n    yhat = model.predict(stackedX)\n    return yhat","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:24:30.655653Z","iopub.execute_input":"2023-04-10T05:24:30.656046Z","iopub.status.idle":"2023-04-10T05:24:30.662483Z","shell.execute_reply.started":"2023-04-10T05:24:30.656014Z","shell.execute_reply":"2023-04-10T05:24:30.661085Z"},"trusted":true},"execution_count":114,"outputs":[]},{"cell_type":"code","source":"yhat = stacked_prediction(members, stacked_model, embeddings_test, test_text)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:00:16.842322Z","iopub.execute_input":"2023-04-10T05:00:16.843418Z","iopub.status.idle":"2023-04-10T05:00:27.394570Z","shell.execute_reply.started":"2023-04-10T05:00:16.843378Z","shell.execute_reply":"2023-04-10T05:00:27.393507Z"},"trusted":true},"execution_count":52,"outputs":[]},{"cell_type":"code","source":"#Only for xgb predictions\nyhat = pd.Series(yhat).map({0 : 'Appreciation',\n                     1 : 'Complaint', \n                     2 : 'Feedback'})","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:03:21.995351Z","iopub.execute_input":"2023-04-10T05:03:21.996396Z","iopub.status.idle":"2023-04-10T05:03:22.007424Z","shell.execute_reply.started":"2023-04-10T05:03:21.996348Z","shell.execute_reply":"2023-04-10T05:03:22.006123Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"yhat = pd.get_dummies(yhat)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:03:24.189212Z","iopub.execute_input":"2023-04-10T05:03:24.189635Z","iopub.status.idle":"2023-04-10T05:03:24.197292Z","shell.execute_reply.started":"2023-04-10T05:03:24.189598Z","shell.execute_reply":"2023-04-10T05:03:24.196001Z"},"trusted":true},"execution_count":56,"outputs":[]},{"cell_type":"code","source":"print(classification_report(test_labels, yhat))","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:03:38.427114Z","iopub.execute_input":"2023-04-10T05:03:38.427797Z","iopub.status.idle":"2023-04-10T05:03:38.447332Z","shell.execute_reply.started":"2023-04-10T05:03:38.427760Z","shell.execute_reply":"2023-04-10T05:03:38.446153Z"},"trusted":true},"execution_count":58,"outputs":[{"name":"stdout","text":"              precision    recall  f1-score   support\n\n           0       0.91      0.89      0.90       310\n           1       0.89      0.90      0.90       639\n           2       0.83      0.81      0.82       246\n\n   micro avg       0.88      0.88      0.88      1195\n   macro avg       0.88      0.87      0.87      1195\nweighted avg       0.88      0.88      0.88      1195\n samples avg       0.88      0.88      0.88      1195\n\n","output_type":"stream"}]},{"cell_type":"code","source":"# loaded_model = tf.keras.models.load_model('/kaggle/input/use-hyper-tuned-model/USE_hyper.hdf5', custom_objects={'KerasLayer': KerasLayer})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Adding predictions","metadata":{}},{"cell_type":"code","source":"predict_df = pd.read_table('/kaggle/input/fb-post-data/FB_posts_unlabeled.txt')","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:05:46.580594Z","iopub.execute_input":"2023-04-10T05:05:46.581326Z","iopub.status.idle":"2023-04-10T05:05:46.603556Z","shell.execute_reply.started":"2023-04-10T05:05:46.581289Z","shell.execute_reply":"2023-04-10T05:05:46.602570Z"},"trusted":true},"execution_count":59,"outputs":[]},{"cell_type":"code","source":"predict_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:05:47.578197Z","iopub.execute_input":"2023-04-10T05:05:47.579285Z","iopub.status.idle":"2023-04-10T05:05:47.590807Z","shell.execute_reply.started":"2023-04-10T05:05:47.579238Z","shell.execute_reply":"2023-04-10T05:05:47.589516Z"},"trusted":true},"execution_count":60,"outputs":[{"execution_count":60,"output_type":"execute_result","data":{"text/plain":"                            postId       message\n0   108381603303_10151119973393304  Love. It. To\n1  115568331790246_371841206162956          NICE\n2  115568331790246_515044031842672      Congrats\n3   147285781446_10151010892176447      Awesome!\n4   159616034235_10150639103634236         Award","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>postId</th>\n      <th>message</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>108381603303_10151119973393304</td>\n      <td>Love. It. To</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>115568331790246_371841206162956</td>\n      <td>NICE</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>115568331790246_515044031842672</td>\n      <td>Congrats</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>147285781446_10151010892176447</td>\n      <td>Awesome!</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>159616034235_10150639103634236</td>\n      <td>Award</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"predict_df['text'] = predict_df['message'].map(lambda text: text.translate(str.maketrans('','',string.punctuation)).strip())\npredict_df['text'] = predict_df['text'].map(lambda text: text.translate(str.maketrans('','',string.digits)).strip())","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:05:48.917811Z","iopub.execute_input":"2023-04-10T05:05:48.918905Z","iopub.status.idle":"2023-04-10T05:05:48.946465Z","shell.execute_reply.started":"2023-04-10T05:05:48.918848Z","shell.execute_reply":"2023-04-10T05:05:48.945501Z"},"trusted":true},"execution_count":61,"outputs":[]},{"cell_type":"code","source":"embeddings_pred = model.encode(predict_df['text'].to_list())","metadata":{"execution":{"iopub.status.busy":"2023-04-10T04:42:43.211172Z","iopub.execute_input":"2023-04-10T04:42:43.211908Z","iopub.status.idle":"2023-04-10T04:42:50.831678Z","shell.execute_reply.started":"2023-04-10T04:42:43.211867Z","shell.execute_reply":"2023-04-10T04:42:50.830533Z"},"trusted":true},"execution_count":33,"outputs":[{"output_type":"display_data","data":{"text/plain":"Batches:   0%|          | 0/64 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c2ff6202b403430da24310ea8638570a"}},"metadata":{}}]},{"cell_type":"code","source":"yhat = stacked_prediction(members, stacked_model, embeddings_pred, predict_df['text'])","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:10.243492Z","iopub.execute_input":"2023-04-10T05:25:10.243874Z","iopub.status.idle":"2023-04-10T05:25:31.030495Z","shell.execute_reply.started":"2023-04-10T05:25:10.243842Z","shell.execute_reply":"2023-04-10T05:25:31.029362Z"},"trusted":true},"execution_count":115,"outputs":[]},{"cell_type":"code","source":"raw_preds = pd.get_dummies(yhat)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:31.032702Z","iopub.execute_input":"2023-04-10T05:25:31.033218Z","iopub.status.idle":"2023-04-10T05:25:31.040953Z","shell.execute_reply.started":"2023-04-10T05:25:31.033180Z","shell.execute_reply":"2023-04-10T05:25:31.039475Z"},"trusted":true},"execution_count":116,"outputs":[]},{"cell_type":"code","source":"raw_preds.sum(axis = 0)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:31.042927Z","iopub.execute_input":"2023-04-10T05:25:31.043881Z","iopub.status.idle":"2023-04-10T05:25:31.054171Z","shell.execute_reply.started":"2023-04-10T05:25:31.043843Z","shell.execute_reply":"2023-04-10T05:25:31.052905Z"},"trusted":true},"execution_count":117,"outputs":[{"execution_count":117,"output_type":"execute_result","data":{"text/plain":"0     548\n1    1048\n2     443\ndtype: int64"},"metadata":{}}]},{"cell_type":"code","source":"raw_preds","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:31.058037Z","iopub.execute_input":"2023-04-10T05:25:31.058438Z","iopub.status.idle":"2023-04-10T05:25:31.069859Z","shell.execute_reply.started":"2023-04-10T05:25:31.058403Z","shell.execute_reply":"2023-04-10T05:25:31.068635Z"},"trusted":true},"execution_count":118,"outputs":[{"execution_count":118,"output_type":"execute_result","data":{"text/plain":"      0  1  2\n0     1  0  0\n1     1  0  0\n2     1  0  0\n3     1  0  0\n4     1  0  0\n...  .. .. ..\n2034  0  1  0\n2035  0  1  0\n2036  0  1  0\n2037  0  1  0\n2038  0  1  0\n\n[2039 rows x 3 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>2034</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2035</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2036</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2037</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2038</th>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>2039 rows × 3 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"preds_df = pd.concat([predict_df['text'], raw_preds], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:31.072113Z","iopub.execute_input":"2023-04-10T05:25:31.072551Z","iopub.status.idle":"2023-04-10T05:25:31.080157Z","shell.execute_reply.started":"2023-04-10T05:25:31.072512Z","shell.execute_reply":"2023-04-10T05:25:31.078961Z"},"trusted":true},"execution_count":119,"outputs":[]},{"cell_type":"code","source":"post_ids = np.array(predict_df['postId'])","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:31.081935Z","iopub.execute_input":"2023-04-10T05:25:31.082498Z","iopub.status.idle":"2023-04-10T05:25:31.089183Z","shell.execute_reply.started":"2023-04-10T05:25:31.082436Z","shell.execute_reply":"2023-04-10T05:25:31.087994Z"},"trusted":true},"execution_count":120,"outputs":[]},{"cell_type":"code","source":"# Don't run before adaptation, this is pseudocode!\nf = open('predictions_stacked_log_v3.csv', 'w')\n# write the first header row\nf.write('postId,Appreciation_pred,Complaint_pred,Feedback_pred' + '\\n')\n\nfor i in range(post_ids.size):\n    Appreciation_pred, Complaint_pred, Feedback_pred = raw_preds.iloc[i,0],  raw_preds.iloc[i,1],  raw_preds.iloc[i,2]\n    if Appreciation_pred not in [0,1] or Complaint_pred not in [0,1] or Feedback_pred not in [0,1]:\n        print('did you forget to turn probability predictions into binary predictions?')\n    if Appreciation_pred + Complaint_pred + Feedback_pred != 1:\n        print('SOMETHING_IS_WRONG')\n    f.write(post_ids[i] + ',' + str(Appreciation_pred) + ',' + str(Complaint_pred) + ',' + str(Feedback_pred) + '\\n')\nf.close()","metadata":{"execution":{"iopub.status.busy":"2023-04-10T05:25:31.091151Z","iopub.execute_input":"2023-04-10T05:25:31.091637Z","iopub.status.idle":"2023-04-10T05:25:31.310797Z","shell.execute_reply.started":"2023-04-10T05:25:31.091594Z","shell.execute_reply":"2023-04-10T05:25:31.309633Z"},"trusted":true},"execution_count":121,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predictions for single models","metadata":{}},{"cell_type":"code","source":"use_model_hyper = tf.keras.models.load_model('/kaggle/input/stacking-models-fb/fb_hyper_use.hdf5', custom_objects={'KerasLayer': KerasLayer, 'f1' : f1})","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:13:28.229636Z","iopub.execute_input":"2023-04-07T06:13:28.230622Z","iopub.status.idle":"2023-04-07T06:13:51.814034Z","shell.execute_reply.started":"2023-04-07T06:13:28.230585Z","shell.execute_reply":"2023-04-07T06:13:51.812503Z"},"trusted":true},"execution_count":79,"outputs":[]},{"cell_type":"code","source":"use_model_hyper.summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:13:52.396987Z","iopub.execute_input":"2023-04-07T06:13:52.397363Z","iopub.status.idle":"2023-04-07T06:13:52.488334Z","shell.execute_reply.started":"2023-04-07T06:13:52.397324Z","shell.execute_reply":"2023-04-07T06:13:52.487137Z"},"trusted":true},"execution_count":80,"outputs":[{"name":"stdout","text":"Model: \"sequential_2\"\n_________________________________________________________________\n Layer (type)                Output Shape              Param #   \n=================================================================\n keras_layer_2 (KerasLayer)  (None, 512)               147354880 \n                                                                 \n dense_6 (Dense)             (None, 384)               196992    \n                                                                 \n dense_7 (Dense)             (None, 384)               147840    \n                                                                 \n dropout_2 (Dropout)         (None, 384)               0         \n                                                                 \n dense_8 (Dense)             (None, 3)                 1155      \n                                                                 \n=================================================================\nTotal params: 147,700,867\nTrainable params: 345,987\nNon-trainable params: 147,354,880\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"raw_preds = use_model_hyper.predict(predict_df['text'])","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:13:52.491007Z","iopub.execute_input":"2023-04-07T06:13:52.491714Z","iopub.status.idle":"2023-04-07T06:14:02.972716Z","shell.execute_reply.started":"2023-04-07T06:13:52.491672Z","shell.execute_reply":"2023-04-07T06:14:02.971681Z"},"trusted":true},"execution_count":81,"outputs":[{"name":"stdout","text":"64/64 [==============================] - 10s 81ms/step\n","output_type":"stream"}]},{"cell_type":"code","source":"from numpy import argmax","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:02.974561Z","iopub.execute_input":"2023-04-07T06:14:02.974920Z","iopub.status.idle":"2023-04-07T06:14:02.980564Z","shell.execute_reply.started":"2023-04-07T06:14:02.974881Z","shell.execute_reply":"2023-04-07T06:14:02.979342Z"},"trusted":true},"execution_count":82,"outputs":[]},{"cell_type":"code","source":"raw_preds","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:02.982369Z","iopub.execute_input":"2023-04-07T06:14:02.982822Z","iopub.status.idle":"2023-04-07T06:14:02.994831Z","shell.execute_reply.started":"2023-04-07T06:14:02.982711Z","shell.execute_reply":"2023-04-07T06:14:02.993831Z"},"trusted":true},"execution_count":83,"outputs":[{"execution_count":83,"output_type":"execute_result","data":{"text/plain":"array([[9.9297792e-01, 6.2972913e-03, 7.2476501e-04],\n       [9.8801893e-01, 1.0550145e-02, 1.4309115e-03],\n       [9.9461645e-01, 4.6169120e-03, 7.6668570e-04],\n       ...,\n       [1.5306695e-01, 5.6700450e-01, 2.7992862e-01],\n       [5.7534602e-02, 9.3357289e-01, 8.8926041e-03],\n       [2.5727996e-01, 7.3755085e-01, 5.1692054e-03]], dtype=float32)"},"metadata":{}}]},{"cell_type":"code","source":"for record in raw_preds:\n    max_index =argmax(record)\n    for i in range(3):\n        if i == max_index:\n            record[i] = 1\n        else:\n            record[i] = 0","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:09.397047Z","iopub.execute_input":"2023-04-07T06:14:09.397761Z","iopub.status.idle":"2023-04-07T06:14:09.416218Z","shell.execute_reply.started":"2023-04-07T06:14:09.397723Z","shell.execute_reply":"2023-04-07T06:14:09.415250Z"},"trusted":true},"execution_count":84,"outputs":[]},{"cell_type":"code","source":"post_ids = np.array(predict_df['postId'])","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:11.204873Z","iopub.execute_input":"2023-04-07T06:14:11.205655Z","iopub.status.idle":"2023-04-07T06:14:11.213279Z","shell.execute_reply.started":"2023-04-07T06:14:11.205614Z","shell.execute_reply":"2023-04-07T06:14:11.212166Z"},"trusted":true},"execution_count":85,"outputs":[]},{"cell_type":"code","source":"raw_preds = raw_preds.astype('int')","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:20.971478Z","iopub.execute_input":"2023-04-07T06:14:20.972253Z","iopub.status.idle":"2023-04-07T06:14:20.977297Z","shell.execute_reply.started":"2023-04-07T06:14:20.972192Z","shell.execute_reply":"2023-04-07T06:14:20.975980Z"},"trusted":true},"execution_count":86,"outputs":[]},{"cell_type":"code","source":"raw_preds","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:34.027989Z","iopub.execute_input":"2023-04-07T06:14:34.029051Z","iopub.status.idle":"2023-04-07T06:14:34.036393Z","shell.execute_reply.started":"2023-04-07T06:14:34.029003Z","shell.execute_reply":"2023-04-07T06:14:34.035135Z"},"trusted":true},"execution_count":87,"outputs":[{"execution_count":87,"output_type":"execute_result","data":{"text/plain":"array([[1, 0, 0],\n       [1, 0, 0],\n       [1, 0, 0],\n       ...,\n       [0, 1, 0],\n       [0, 1, 0],\n       [0, 1, 0]])"},"metadata":{}}]},{"cell_type":"code","source":"# Don't run before adaptation, this is pseudocode!\nf = open('predictions_USE_hyper.csv', 'w')\n# write the first header row\nf.write('postId,Appreciation_pred,Complaint_pred,Feedback_pred' + '\\n')\n\nfor i in range(post_ids.size):\n    Appreciation_pred, Complaint_pred, Feedback_pred = raw_preds[i][0], raw_preds[i][1], raw_preds[i][2]\n    if Appreciation_pred not in [0,1] or Complaint_pred not in [0,1] or Feedback_pred not in [0,1]:\n        print('did you forget to turn probability predictions into binary predictions?')\n    if Appreciation_pred + Complaint_pred + Feedback_pred != 1:\n        print('SOMETHING_IS_WRONG')\n    f.write(post_ids[i] + ',' + str(Appreciation_pred) + ',' + str(Complaint_pred) + ',' + str(Feedback_pred) + '\\n')\nf.close()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T06:14:51.190806Z","iopub.execute_input":"2023-04-07T06:14:51.191420Z","iopub.status.idle":"2023-04-07T06:14:51.210336Z","shell.execute_reply.started":"2023-04-07T06:14:51.191380Z","shell.execute_reply":"2023-04-07T06:14:51.209240Z"},"trusted":true},"execution_count":88,"outputs":[]},{"cell_type":"markdown","source":"### Loading model","metadata":{}}]}