Topic_analysis_bertopic_take2_speeches.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "47943d3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-20 17:46:05.263751: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-20 17:46:05.368995: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2023-04-20 17:46:05.369010: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2023-04-20 17:46:05.395408: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2023-04-20 17:46:05.969784: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-20 17:46:05.969918: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-20 17:46:05.969926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "2023-04-20 17:46:07.973312: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2023-04-20 17:46:07.973329: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-04-20 17:46:07.973359: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (muddy-HP-ProDesk-600-G3-SFF): /proc/driver/nvidia/version does not exist\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import re\n",
    "import unicodedata\n",
    "from bertopic import BERTopic\n",
    "import spacy\n",
    "import en_core_web_md\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction import text\n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import plotly.express as px\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3fc2fd64",
   "metadata": {},
   "outputs": [],
   "source": [
    "amrhet_path = ['./Data/speeches/']\n",
    "all_paths = ['./Data/amrhet/', './NYT/', './Data/WSJ/', './Data/GWB/', './Data/speeches/', './Data/Top10/']\n",
    "\n",
    "def load_docs(paths):\n",
    "    if not type(paths) == list:\n",
    "        paths = list(paths)\n",
    "    speeches = []\n",
    "    for path in paths:\n",
    "        list_of_files = []\n",
    "        for root, dirs, files in os.walk(path):\n",
    "            for file in files:\n",
    "                if file.endswith('.txt'):\n",
    "                    list_of_files.append(os.path.join(root,file))\n",
    "\n",
    "        for file in list_of_files:\n",
    "            with open(file, encoding='utf-8') as f:\n",
    "                text = f.read()\n",
    "            f.close()\n",
    "            speeches.append(text)\n",
    "\n",
    "    #clean out goofy unicode  space characters \n",
    "    speeches = [unicodedata.normalize(\"NFKD\", speech) for speech in speeches if len(speech)>0 ]\n",
    "    #clean out xa0 space characters\n",
    "    [speech.replace(u'\\xa0', '') for speech in speeches]; # ; supresses output\n",
    "    # remove [stuff] in between square brackets\n",
    "    def remove_bracket(text):\n",
    "        return re.sub(r'(\\[[^w]*\\]\\s)', '',text)\n",
    "    speeches = [remove_bracket(speech) for speech in speeches]\n",
    "    # Clean up whitespace\n",
    "    speeches = [re.sub(r'[\\s+]', ' ', speech) for speech in speeches]\n",
    "    return(speeches, list_of_files)\n",
    "\n",
    "all_speeches, all_files = load_docs(all_paths)\n",
    "amrhet, amrhet_files = load_docs(amrhet_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5e3d6ba1",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_md')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad1db239",
   "metadata": {},
   "source": [
    "<A HREF=\"https://maartengr.github.io/BERTopic/index.html#quick-start\">Bertopic quick start</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7be26616",
   "metadata": {},
   "outputs": [],
   "source": [
    "kaggle_file = open(\"./Data/word_lists/kaggle_stopwords.txt\", \"r\")\n",
    "kaggle_data = kaggle_file.read()\n",
    "kaggle_set = set([word for word in kaggle_data.split('\\n')])\n",
    "kaggle_file.close()\n",
    "stop_words = list(text.ENGLISH_STOP_WORDS.union(kaggle_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c599653",
   "metadata": {},
   "source": [
    "<A HREF=\"https://maartengr.github.io/BERTopic/getting_started/parameter%20tuning/parametertuning.html\">BERTopic parameter tuning</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f738667c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use CountVectorizer as a parameter for BERTopic to remove stop words\n",
    "vectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words=stop_words)\n",
    "# nr_topics = \"auto\" combines two topics if similarity is 90%\n",
    "topic_model = BERTopic(vectorizer_model=vectorizer_model,nr_topics=9,top_n_words=10)\n",
    "topic_model.fit(amrhet) # learn model on one set of docs\n",
    "topics, probs = topic_model.transform(amrhet) # predict topics for only subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "886dbd7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic</th>\n",
       "      <th>Count</th>\n",
       "      <th>Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>42</td>\n",
       "      <td>-1_people_going_know_want</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0_jobs_americans_new_people</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1_people_iraq_security_united</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>2_people_america_new_world</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Topic  Count                           Name\n",
       "0     -1     42      -1_people_going_know_want\n",
       "1      0     16    0_jobs_americans_new_people\n",
       "2      1     17  1_people_iraq_security_united\n",
       "3      2     26     2_people_america_new_world"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_model.get_topic_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "810d71c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "marker": {
          "color": "#D55E00"
         },
         "orientation": "h",
         "type": "bar",
         "x": [
          0.013122299638296739,
          0.014307867034013984,
          0.014449917709919846,
          0.014573449435628973,
          0.015572253236196458,
          0.015713424665639263,
          0.016068663982115226,
          0.017209674079531945,
          0.017500895880742832,
          0.018509906809437877
         ],
         "xaxis": "x",
         "y": [
          "american  ",
          "economy  ",
          "congress  ",
          "year  ",
          "america  ",
          "work  ",
          "people  ",
          "new  ",
          "americans  ",
          "jobs  "
         ],
         "yaxis": "y"
        },
        {
         "marker": {
          "color": "#0072B2"
         },
         "orientation": "h",
         "type": "bar",
         "x": [
          0.013609646941256834,
          0.013727673255932361,
          0.013755707286514646,
          0.01389821692704743,
          0.01393913717246245,
          0.01519986092750512,
          0.01526076635839756,
          0.016191812912429094,
          0.018481829224239143,
          0.020864824588706074
         ],
         "xaxis": "x2",
         "y": [
          "government  ",
          "weve  ",
          "world  ",
          "states  ",
          "american  ",
          "war  ",
          "united  ",
          "security  ",
          "iraq  ",
          "people  "
         ],
         "yaxis": "y2"
        },
        {
         "marker": {
          "color": "#CC79A7"
         },
         "orientation": "h",
         "type": "bar",
         "x": [
          0.010694460566673291,
          0.010736573171115489,
          0.011658813265314259,
          0.011812713305163368,
          0.012456094667016806,
          0.012520697881097718,
          0.013662651745537727,
          0.015431679238717452,
          0.016261200514039245,
          0.018702963136556647
         ],
         "xaxis": "x3",
         "y": [
          "states  ",
          "country  ",
          "american  ",
          "time  ",
          "know  ",
          "years  ",
          "world  ",
          "new  ",
          "america  ",
          "people  "
         ],
         "yaxis": "y3"
        }
       ],
       "layout": {
        "annotations": [
         {
          "font": {
           "size": 16
          },
          "showarrow": false,
          "text": "Topic 0",
          "x": 0.0875,
          "xanchor": "center",
          "xref": "paper",
          "y": 1,
          "yanchor": "bottom",
          "yref": "paper"
         },
         {
          "font": {
           "size": 16
          },
          "showarrow": false,
          "text": "Topic 1",
          "x": 0.36250000000000004,
          "xanchor": "center",
          "xref": "paper",
          "y": 1,
          "yanchor": "bottom",
          "yref": "paper"
         },
         {
          "font": {
           "size": 16
          },
          "showarrow": false,
          "text": "Topic 2",
          "x": 0.6375000000000001,
          "xanchor": "center",
          "xref": "paper",
          "y": 1,
          "yanchor": "bottom",
          "yref": "paper"
         }
        ],
        "height": 390,
        "hoverlabel": {
         "bgcolor": "white",
         "font": {
          "family": "Rockwell",
          "size": 16
         }
        },
        "showlegend": false,
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "#C8D4E3",
             "linecolor": "#C8D4E3",
             "minorgridcolor": "#C8D4E3",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "#C8D4E3",
             "linecolor": "#C8D4E3",
             "minorgridcolor": "#C8D4E3",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "white",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "#C8D4E3"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "white",
          "polar": {
           "angularaxis": {
            "gridcolor": "#EBF0F8",
            "linecolor": "#EBF0F8",
            "ticks": ""
           },
           "bgcolor": "white",
           "radialaxis": {
            "gridcolor": "#EBF0F8",
            "linecolor": "#EBF0F8",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "white",
            "gridcolor": "#DFE8F3",
            "gridwidth": 2,
            "linecolor": "#EBF0F8",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "#EBF0F8"
           },
           "yaxis": {
            "backgroundcolor": "white",
            "gridcolor": "#DFE8F3",
            "gridwidth": 2,
            "linecolor": "#EBF0F8",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "#EBF0F8"
           },
           "zaxis": {
            "backgroundcolor": "white",
            "gridcolor": "#DFE8F3",
            "gridwidth": 2,
            "linecolor": "#EBF0F8",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "#EBF0F8"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "#DFE8F3",
            "linecolor": "#A2B1C6",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "#DFE8F3",
            "linecolor": "#A2B1C6",
            "ticks": ""
           },
           "bgcolor": "white",
           "caxis": {
            "gridcolor": "#DFE8F3",
            "linecolor": "#A2B1C6",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "#EBF0F8",
           "linecolor": "#EBF0F8",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "#EBF0F8",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "#EBF0F8",
           "linecolor": "#EBF0F8",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "#EBF0F8",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "font": {
          "color": "Black",
          "size": 22
         },
         "text": "Topic Word Scores",
         "x": 0.5,
         "xanchor": "center",
         "yanchor": "top"
        },
        "width": 1000,
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          0.175
         ],
         "showgrid": true
        },
        "xaxis2": {
         "anchor": "y2",
         "domain": [
          0.275,
          0.45
         ],
         "showgrid": true
        },
        "xaxis3": {
         "anchor": "y3",
         "domain": [
          0.55,
          0.7250000000000001
         ],
         "showgrid": true
        },
        "xaxis4": {
         "anchor": "y4",
         "domain": [
          0.825,
          1
         ],
         "showgrid": true
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "showgrid": true
        },
        "yaxis2": {
         "anchor": "x2",
         "domain": [
          0,
          1
         ],
         "showgrid": true
        },
        "yaxis3": {
         "anchor": "x3",
         "domain": [
          0,
          1
         ],
         "showgrid": true
        },
        "yaxis4": {
         "anchor": "x4",
         "domain": [
          0,
          1
         ],
         "showgrid": true
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"3485f022-7106-4686-a121-a076c5ddab83\" class=\"plotly-graph-div\" style=\"height:390.0px; width:1000px;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3485f022-7106-4686-a121-a076c5ddab83\")) {                    Plotly.newPlot(                        \"3485f022-7106-4686-a121-a076c5ddab83\",                        [{\"marker\":{\"color\":\"#D55E00\"},\"orientation\":\"h\",\"x\":[0.013122299638296739,0.014307867034013984,0.014449917709919846,0.014573449435628973,0.015572253236196458,0.015713424665639263,0.016068663982115226,0.017209674079531945,0.017500895880742832,0.018509906809437877],\"y\":[\"american  \",\"economy  \",\"congress  \",\"year  \",\"america  \",\"work  \",\"people  \",\"new  \",\"americans  \",\"jobs  \"],\"type\":\"bar\",\"xaxis\":\"x\",\"yaxis\":\"y\"},{\"marker\":{\"color\":\"#0072B2\"},\"orientation\":\"h\",\"x\":[0.013609646941256834,0.013727673255932361,0.013755707286514646,0.01389821692704743,0.01393913717246245,0.01519986092750512,0.01526076635839756,0.016191812912429094,0.018481829224239143,0.020864824588706074],\"y\":[\"government  \",\"weve  \",\"world  \",\"states  \",\"american  \",\"war  \",\"united  \",\"security  \",\"iraq  \",\"people  \"],\"type\":\"bar\",\"xaxis\":\"x2\",\"yaxis\":\"y2\"},{\"marker\":{\"color\":\"#CC79A7\"},\"orientation\":\"h\",\"x\":[0.010694460566673291,0.010736573171115489,0.011658813265314259,0.011812713305163368,0.012456094667016806,0.012520697881097718,0.013662651745537727,0.015431679238717452,0.016261200514039245,0.018702963136556647],\"y\":[\"states  \",\"country  \",\"american  \",\"time  \",\"know  \",\"years  \",\"world  \",\"new  \",\"america  \",\"people  \"],\"type\":\"bar\",\"xaxis\":\"x3\",\"yaxis\":\"y3\"}],                        {\"template\":{\"data\":{\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"#C8D4E3\",\"linecolor\":\"#C8D4E3\",\"minorgridcolor\":\"#C8D4E3\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"#C8D4E3\",\"linecolor\":\"#C8D4E3\",\"minorgridcolor\":\"#C8D4E3\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"choropleth\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"contourcarpet\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"contour\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"heatmapgl\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"heatmap\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"histogram2dcontour\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"histogram2d\"}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scattermapbox\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterpolargl\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterpolar\"}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]],\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"white\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"#C8D4E3\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"white\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"#EBF0F8\",\"linecolor\":\"#EBF0F8\",\"ticks\":\"\"},\"bgcolor\":\"white\",\"radialaxis\":{\"gridcolor\":\"#EBF0F8\",\"linecolor\":\"#EBF0F8\",\"ticks\":\"\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"#DFE8F3\",\"gridwidth\":2,\"linecolor\":\"#EBF0F8\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"#EBF0F8\"},\"yaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"#DFE8F3\",\"gridwidth\":2,\"linecolor\":\"#EBF0F8\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"#EBF0F8\"},\"zaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"#DFE8F3\",\"gridwidth\":2,\"linecolor\":\"#EBF0F8\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"#EBF0F8\"}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"ternary\":{\"aaxis\":{\"gridcolor\":\"#DFE8F3\",\"linecolor\":\"#A2B1C6\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"#DFE8F3\",\"linecolor\":\"#A2B1C6\",\"ticks\":\"\"},\"bgcolor\":\"white\",\"caxis\":{\"gridcolor\":\"#DFE8F3\",\"linecolor\":\"#A2B1C6\",\"ticks\":\"\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"#EBF0F8\",\"linecolor\":\"#EBF0F8\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"#EBF0F8\",\"zerolinewidth\":2},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"#EBF0F8\",\"linecolor\":\"#EBF0F8\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"#EBF0F8\",\"zerolinewidth\":2}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.175],\"showgrid\":true},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"showgrid\":true},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.275,0.45],\"showgrid\":true},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"showgrid\":true},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.55,0.7250000000000001],\"showgrid\":true},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"showgrid\":true},\"xaxis4\":{\"anchor\":\"y4\",\"domain\":[0.825,1.0],\"showgrid\":true},\"yaxis4\":{\"anchor\":\"x4\",\"domain\":[0.0,1.0],\"showgrid\":true},\"annotations\":[{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Topic 0\",\"x\":0.0875,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Topic 1\",\"x\":0.36250000000000004,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Topic 2\",\"x\":0.6375000000000001,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"title\":{\"font\":{\"size\":22,\"color\":\"Black\"},\"text\":\"Topic Word Scores\",\"x\":0.5,\"xanchor\":\"center\",\"yanchor\":\"top\"},\"hoverlabel\":{\"font\":{\"size\":16,\"family\":\"Rockwell\"},\"bgcolor\":\"white\"},\"showlegend\":false,\"width\":1000,\"height\":390.0},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('3485f022-7106-4686-a121-a076c5ddab83');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "topic_model.visualize_barchart(n_words=10, height=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09456b07",
   "metadata": {},
   "source": [
    "<A HREF=\"https://maartengr.github.io/BERTopic/api/bertopic.html\">BERTopic API</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62d857cb",
   "metadata": {},
   "source": [
    "<A HREF=\"https://towardsdatascience.com/interactive-topic-modeling-with-bertopic-1ea55e7d73d8\">Add tf-idf to BERTopics</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4291ef88",
   "metadata": {},
   "source": [
    "## I'm starting to like this one, with TF-IDF!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "271b5a9c",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "max_df corresponds to < documents than min_df",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 8\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# Model\u001b[39;00m\n\u001b[1;32m      7\u001b[0m model_tfidf \u001b[38;5;241m=\u001b[39m BERTopic(vectorizer_model\u001b[38;5;241m=\u001b[39mvectorizer_tfidf,nr_topics\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m9\u001b[39m)\n\u001b[0;32m----> 8\u001b[0m topics_tfidf, probs_tfidf \u001b[38;5;241m=\u001b[39m \u001b[43mmodel_tfidf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamrhet\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43membeddings\u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m      9\u001b[0m topic_df \u001b[38;5;241m=\u001b[39m model_tfidf\u001b[38;5;241m.\u001b[39mget_topic_info()\n\u001b[1;32m     10\u001b[0m topic_df\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/bertopic/_bertopic.py:366\u001b[0m, in \u001b[0;36mBERTopic.fit_transform\u001b[0;34m(self, documents, embeddings, y)\u001b[0m\n\u001b[1;32m    363\u001b[0m     documents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sort_mappings_by_frequency(documents)\n\u001b[1;32m    365\u001b[0m \u001b[38;5;66;03m# Extract topics by calculating c-TF-IDF\u001b[39;00m\n\u001b[0;32m--> 366\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_extract_topics\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    368\u001b[0m \u001b[38;5;66;03m# Reduce topics\u001b[39;00m\n\u001b[1;32m    369\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnr_topics:\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/bertopic/_bertopic.py:2980\u001b[0m, in \u001b[0;36mBERTopic._extract_topics\u001b[0;34m(self, documents)\u001b[0m\n\u001b[1;32m   2971\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\" Extract topics from the clusters using a class-based TF-IDF\u001b[39;00m\n\u001b[1;32m   2972\u001b[0m \n\u001b[1;32m   2973\u001b[0m \u001b[38;5;124;03mArguments:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2977\u001b[0m \u001b[38;5;124;03m    c_tf_idf: The resulting matrix giving a value (importance score) for each word per topic\u001b[39;00m\n\u001b[1;32m   2978\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   2979\u001b[0m documents_per_topic \u001b[38;5;241m=\u001b[39m documents\u001b[38;5;241m.\u001b[39mgroupby([\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTopic\u001b[39m\u001b[38;5;124m'\u001b[39m], as_index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\u001b[38;5;241m.\u001b[39magg({\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDocument\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin})\n\u001b[0;32m-> 2980\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mc_tf_idf_, words \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_c_tf_idf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments_per_topic\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2981\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtopic_representations_ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_extract_words_per_topic(words, documents)\n\u001b[1;32m   2982\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_topic_vectors()\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/bertopic/_bertopic.py:3121\u001b[0m, in \u001b[0;36mBERTopic._c_tf_idf\u001b[0;34m(self, documents_per_topic, fit, partial_fit)\u001b[0m\n\u001b[1;32m   3119\u001b[0m     X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvectorizer_model\u001b[38;5;241m.\u001b[39mpartial_fit(documents)\u001b[38;5;241m.\u001b[39mupdate_bow(documents)\n\u001b[1;32m   3120\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m fit:\n\u001b[0;32m-> 3121\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvectorizer_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3122\u001b[0m     X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvectorizer_model\u001b[38;5;241m.\u001b[39mtransform(documents)\n\u001b[1;32m   3123\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:2103\u001b[0m, in \u001b[0;36mTfidfVectorizer.fit\u001b[0;34m(self, raw_documents, y)\u001b[0m\n\u001b[1;32m   2096\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_warn_for_unused_params()\n\u001b[1;32m   2097\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tfidf \u001b[38;5;241m=\u001b[39m TfidfTransformer(\n\u001b[1;32m   2098\u001b[0m     norm\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnorm,\n\u001b[1;32m   2099\u001b[0m     use_idf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_idf,\n\u001b[1;32m   2100\u001b[0m     smooth_idf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msmooth_idf,\n\u001b[1;32m   2101\u001b[0m     sublinear_tf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msublinear_tf,\n\u001b[1;32m   2102\u001b[0m )\n\u001b[0;32m-> 2103\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_documents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2104\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tfidf\u001b[38;5;241m.\u001b[39mfit(X)\n\u001b[1;32m   2105\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:1398\u001b[0m, in \u001b[0;36mCountVectorizer.fit_transform\u001b[0;34m(self, raw_documents, y)\u001b[0m\n\u001b[1;32m   1396\u001b[0m min_doc_count \u001b[38;5;241m=\u001b[39m min_df \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(min_df, Integral) \u001b[38;5;28;01melse\u001b[39;00m min_df \u001b[38;5;241m*\u001b[39m n_doc\n\u001b[1;32m   1397\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m max_doc_count \u001b[38;5;241m<\u001b[39m min_doc_count:\n\u001b[0;32m-> 1398\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmax_df corresponds to < documents than min_df\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   1399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m max_features \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1400\u001b[0m     X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sort_features(X, vocabulary)\n",
      "\u001b[0;31mValueError\u001b[0m: max_df corresponds to < documents than min_df"
     ]
    }
   ],
   "source": [
    "# Create TF-IDF sparse matrix\n",
    "# tf-idf embeddings and documents must be same length, thus have to use same set of docs for both\n",
    "vectorizer_tfidf = TfidfVectorizer(min_df=2, max_df=0.7, stop_words=stop_words)\n",
    "embeddings = vectorizer_tfidf.fit_transform(amrhet)\n",
    "\n",
    "# Model\n",
    "model_tfidf = BERTopic(vectorizer_model=vectorizer_tfidf,nr_topics=9)\n",
    "topics_tfidf, probs_tfidf = model_tfidf.fit_transform(amrhet, embeddings) \n",
    "topic_df = model_tfidf.get_topic_info()\n",
    "topic_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a889be35",
   "metadata": {},
   "source": [
    "<A HREF=\"https://maartengr.github.io/BERTopic/api/plotting/barchart.html\">BERTopic plotting</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f98e6af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tfidf.visualize_barchart(n_words=10, height=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "595e81c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_topics = model_tfidf.get_topics()\n",
    "all_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1665c30",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_df['topic_name'] = ['other', 'intl relations', 'terrorism', 'tragedy', 'health care',\n",
    "                          'civil rights', 'economy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f765cd2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'text': amrhet, 'file': amrhet_files, 'Topic': topics_tfidf})\n",
    "df = pd.merge(df, topic_df[['Topic', 'topic_name']], how='left', on='Topic')\n",
    "\n",
    "meta_info = pd.read_csv('./Data/genData/date_file_text_speeches_oba.csv')\n",
    "meta_info = meta_info.drop('text', axis=1)\n",
    "meta_info.date = pd.to_datetime(meta_info.date, format='%Y-%m-%d')\n",
    "df = pd.merge(df, meta_info, how='inner', on='file')\n",
    "\n",
    "sp_enc = pd.read_csv('./Data/genData/speeches_spacy_encodings.csv')\n",
    "sp_enc.date = pd.to_datetime(sp_enc.date, format='%Y-%m-%d')\n",
    "label_data = pd.merge(sp_enc, df, how='left', on='date')\n",
    "pca_data = sp_enc.drop('date', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "856c99fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 2)\n",
    "pcafit = pca.fit_transform(pca_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df0a594c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca.explained_variance_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "626e7ebe",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter(pcafit, \n",
    "                 x=0, \n",
    "                 y=1, \n",
    "                 color=df.topic_name,\n",
    "                 hover_name=df.date,\n",
    "                 labels = {'0':'PC1', '1':'PC2', 'color':'topic_name'},\n",
    "                 color_discrete_sequence=px.colors.qualitative.Bold)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a747425",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.to_csv('./Data/genData/bertopic_topics_amrhet.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4daa90d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_dict = dict(topic_df.topic_name[1:].reset_index().drop('index', axis=1).topic_name)\n",
    "#model_tfidf.set_topic_labels(topic_dict)\n",
    "model_tfidf.visualize_documents(amrhet, embeddings=embeddings, custom_labels=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e59ee262",
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer = umap.UMAP()\n",
    "umap_df = reducer.fit_transform(pca_data)\n",
    "umap_df = pd.DataFrame(umap_df)\n",
    "umap_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b1ec689",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter(umap_df, \n",
    "                 x=0, \n",
    "                 y=1, \n",
    "                 color=df.topic_name,\n",
    "                 hover_name=df.date,\n",
    "                 labels = {'0':'PC1', '1':'PC2', 'color':'topic_name'},\n",
    "                 color_discrete_sequence=px.colors.qualitative.Bold)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3097f44",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}