1_summarise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note:\n",
    "\n",
    "This file was used for initial exploration and is saved for reference, the logic here has been formalised in the `create_summaries.py` file. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plan\n",
    "\n",
    "1. Use GPT to summarise full doc with map-reduce method\n",
    "2. Use Davinci to summarise full doc with map-reduce method\n",
    "3. Use GPT to summarise sections by title\n",
    "4. Try Vector Summarisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from dotenv import load_dotenv\n",
    "from langchain.document_loaders import Docx2txtLoader\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.chains.combine_documents.stuff import StuffDocumentsChain\n",
    "from langchain.chains import ReduceDocumentsChain, MapReduceDocumentsChain\n",
    "from langchain.chains.llm import LLMChain\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.llms import OpenAI\n",
    "from typing import Optional\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "from typing import Optional, Union\n",
    "from langchain.schema import Document\n",
    "\n",
    "from langchain.vectorstores import FAISS\n",
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "import plotly.graph_objects as go\n",
    "from langchain.chains.summarize import load_summarize_chain\n",
    "\n",
    "# Data Science\n",
    "import numpy as np\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SUMMARY_SAVE_DIR = Path(\"data\", \"summaries\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = Path(\"data\", \"raw\")\n",
    "toc_2015_fname = Path(DATA_DIR, \"Jan 2015.docx\")\n",
    "toc_2023_fname = Path(DATA_DIR, \"Mar 2023.docx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader_2015 = Docx2txtLoader(str(toc_2015_fname))  # str reqd for loader\n",
    "data_2015 = loader_2015.load()\n",
    "loader_2023 = Docx2txtLoader(str(toc_2023_fname))\n",
    "data_2023 = loader_2023.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Map reduce template"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Map\n",
    "map_template = \"\"\"\n",
    "Write a concise summary of the following:\n",
    "\"{docs}\"\n",
    "CONCISE SUMMARY:\n",
    "\"\"\"\n",
    "map_prompt = PromptTemplate.from_template(map_template)\n",
    "# Reduce\n",
    "reduce_template = \"\"\"Write a concise summary of the following text delimited by triple backquotes.\n",
    "Return your response in bullet points which covers the key points of the text.\n",
    "```{doc_summaries}```\n",
    "BULLET POINT SUMMARY:\"\"\"\n",
    "reduce_prompt = PromptTemplate.from_template(reduce_template)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Whole Document - GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt4 = ChatOpenAI(model_name=\"gpt-4\")\n",
    "# gpt4 has up to 8,192 tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015 T&Cs has been split into 5 docs. Made up of [3693, 3626, 3732, 3771, 3738] length tokens.\n",
      "2023 T&Cs has been split into 3 docs.Made up of [3601, 3479, 3199] length tokens.\n"
     ]
    }
   ],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=4000, chunk_overlap=0\n",
    ")\n",
    "split_2015 = text_splitter.split_documents(data_2015)\n",
    "split_2023 = text_splitter.split_documents(data_2023)\n",
    "\n",
    "token_lengths_2015 = [\n",
    "    gpt4.get_num_tokens(split_2015[i].page_content) for i in range(len(split_2015))\n",
    "]\n",
    "\n",
    "token_lengths_2023 = [\n",
    "    gpt4.get_num_tokens(split_2023[i].page_content) for i in range(len(split_2023))\n",
    "]\n",
    "print(\n",
    "    f\"2015 T&Cs has been split into {len(split_2015)} docs. Made up of {token_lengths_2015} length tokens.\"\n",
    ")\n",
    "print(\n",
    "    f\"2023 T&Cs has been split into {len(split_2023)} docs.Made up of {token_lengths_2023} length tokens.\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Map Reduce Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_chain = LLMChain(llm=gpt4, prompt=map_prompt)\n",
    "\n",
    "# Run chain\n",
    "\n",
    "reduce_chain = LLMChain(llm=gpt4, prompt=reduce_prompt)\n",
    "\n",
    "# Takes a list of documents, combines them into a single string, and passes this to an LLMChain\n",
    "combine_documents_chain = StuffDocumentsChain(\n",
    "    llm_chain=reduce_chain, document_variable_name=\"doc_summaries\"\n",
    ")\n",
    "\n",
    "# Combines and iteravely reduces the mapped documents\n",
    "reduce_documents_chain = ReduceDocumentsChain(\n",
    "    # This is final chain that is called.\n",
    "    combine_documents_chain=combine_documents_chain,\n",
    "    # If documents exceed context for `StuffDocumentsChain`\n",
    "    collapse_documents_chain=combine_documents_chain,\n",
    "    # The maximum number of tokens to group documents into.\n",
    "    token_max=4000,\n",
    ")\n",
    "\n",
    "# Combining documents by mapping a chain over them, then combining results\n",
    "map_reduce_chain = MapReduceDocumentsChain(\n",
    "    # Map chain\n",
    "    llm_chain=map_chain,\n",
    "    # Reduce chain\n",
    "    reduce_documents_chain=reduce_documents_chain,\n",
    "    # The variable name in the llm_chain to put the documents in\n",
    "    document_variable_name=\"docs\",\n",
    "    # Return the results of the map steps in the output\n",
    "    return_intermediate_steps=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "RUN_MAP_REDUCE_GPT = False  # Flag to reduce running time + $$\n",
    "if RUN_MAP_REDUCE_GPT:\n",
    "    output_map_reduce_15 = map_reduce_chain.run(split_2015)\n",
    "    output_map_reduce_23 = map_reduce_chain.run(split_2023)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE = False\n",
    "if SAVE:\n",
    "    with open(Path(SUMMARY_SAVE_DIR, \"gpt4_map_reduce_summarized_2015.txt\"), \"w\") as f:\n",
    "        f.write(output_map_reduce_15)\n",
    "    with open(Path(SUMMARY_SAVE_DIR, \"gpt4_map_reduce_summarized_2023.txt\"), \"w\") as f:\n",
    "        f.write(output_map_reduce_23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_MAP_REDUCE_GPT:\n",
    "    print(f\"2015 map-reduce summary contains {len(output_map_reduce_15.split())} words\")\n",
    "    print(f\"2023 map-reduce summary contains {len(output_map_reduce_23.split())} words\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Whole Document - Davinci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=2000, chunk_overlap=0\n",
    ")\n",
    "split_2015 = text_splitter.split_documents(data_2015)\n",
    "split_2023 = text_splitter.split_documents(data_2023)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "davinci = OpenAI(temperature=0, model=\"text-davinci-003\", max_tokens=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map Reduce Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_chain = LLMChain(llm=davinci, prompt=map_prompt)\n",
    "reduce_chain = LLMChain(llm=davinci, prompt=reduce_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "RUN_MAP_REDUCE_DAVINCI = False  # Flag to reduce running time + $$\n",
    "if RUN_MAP_REDUCE_DAVINCI:\n",
    "    output_map_reduce_davinci_15 = map_reduce_chain.run(split_2015)\n",
    "    output_map_reduce_davinci_23 = map_reduce_chain.run(split_2023)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if SAVE:\n",
    "    with open(\n",
    "        Path(SUMMARY_SAVE_DIR, \"davinci_map_reduce_summarized_2015.txt\"), \"w\"\n",
    "    ) as f:\n",
    "        f.write(output_map_reduce_davinci_15)\n",
    "    with open(\n",
    "        Path(SUMMARY_SAVE_DIR, \"davinci_map_reduce_summarized_2023.txt\"), \"w\"\n",
    "    ) as f:\n",
    "        f.write(output_map_reduce_davinci_23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_MAP_REDUCE_DAVINCI:\n",
    "    print(\n",
    "        f\"2015 davinci map-reduce summary contains {len(output_map_reduce_davinci_15.split())} words\"\n",
    "    )\n",
    "    print(\n",
    "        f\"2023 davinci map-reduce summary contains {len(output_map_reduce_davinci_23.split())} words\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarising by section\n",
    "\n",
    "Manually splitting documents into sections that make sense to a human, before summarising. Hope is that more of the key information will be maintained, however this would be unsustainable for very long corpora. \n",
    "\n",
    "## Splitting docs by sections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_text_2015 = data_2015[0].page_content\n",
    "raw_text_2023 = data_2023[0].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015 T&S contain 15399 words\n",
      "2023 T&S contain 8386 words\n"
     ]
    }
   ],
   "source": [
    "print(f\"2015 T&S contain {len(raw_text_2015.split())} words\")\n",
    "print(f\"2023 T&S contain {len(raw_text_2023.split())} words\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GPT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2015 Sections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "delimiters_2015 = [\n",
    "    \"A. ITUNES STORE, MAC APP STORE, APP STORE AND IBOOKS STORE TERMS OF SALE\",\n",
    "    \"B. ITUNES STORE TERMS AND CONDITIONS\",\n",
    "    \"C. MAC APP STORE, APP STORE AND IBOOKS STORE TERMS AND CONDITIONS\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_text_2015 = raw_text_2015[\n",
    "    200:\n",
    "]  # removing the initial text as this matches the delimiters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def string_between_substrings(input_str: str, start: str, end: Optional[str]):\n",
    "    if end is None:\n",
    "        return (input_str.split(start))[1]\n",
    "    return (input_str.split(start))[1].split(end)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_section = string_between_substrings(\n",
    "    raw_text_2015, delimiters_2015[0], delimiters_2015[1]\n",
    ")\n",
    "\n",
    "b_section = string_between_substrings(\n",
    "    raw_text_2015, delimiters_2015[1], delimiters_2015[2]\n",
    ")\n",
    "\n",
    "c_section = string_between_substrings(raw_text_2015, delimiters_2015[2], None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A Section length: 2280 words,\n",
      "      B Section length: 5186 words,\n",
      "      C Section length: 7767 words \n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    f\"\"\"A Section length: {len(a_section.split())} words,\n",
    "      B Section length: {len(b_section.split())} words,\n",
    "      C Section length: {len(c_section.split())} words \"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=4000, chunk_overlap=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_doc_2015 = text_splitter.create_documents([a_section])\n",
    "b_doc_2015 = text_splitter.create_documents([b_section])\n",
    "c_doc_2015 = text_splitter.create_documents([c_section])\n",
    "\n",
    "a_split_2015 = text_splitter.split_documents(a_doc_2015)\n",
    "b_split_2015 = text_splitter.split_documents(b_doc_2015)\n",
    "c_split_2015 = text_splitter.split_documents(c_doc_2015)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_chain = LLMChain(llm=gpt4, prompt=map_prompt)\n",
    "reduce_chain = LLMChain(llm=gpt4, prompt=reduce_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "RUN_SECTIONS_MAP_REDUCE = False\n",
    "if RUN_SECTIONS_MAP_REDUCE:\n",
    "    output_map_reduce_15_a = map_reduce_chain.run(a_split_2015)\n",
    "    output_map_reduce_15_b = map_reduce_chain.run(b_split_2015)\n",
    "    output_map_reduce_15_c = map_reduce_chain.run(c_split_2015)\n",
    "\n",
    "    sections_2015_summary = (\n",
    "        output_map_reduce_15_a\n",
    "        + \"\\n\"\n",
    "        + output_map_reduce_15_b\n",
    "        + \"\\n\"\n",
    "        + output_map_reduce_15_c\n",
    "    )\n",
    "SAVE_SECTIONS_MAP_REDUCE = False\n",
    "if RUN_SECTIONS_MAP_REDUCE and SAVE_SECTIONS_MAP_REDUCE:\n",
    "    with open(Path(SUMMARY_SAVE_DIR, \"2015_sections_summary.txt\"), \"w\") as f:\n",
    "        f.write(sections_2015_summary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2023 Sections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "delimiters_2023 = [\n",
    "    \"A. INTRODUCTION TO OUR SERVICES\",\n",
    "    # \"B. USING OUR SERVICES\",\n",
    "    \"C. YOUR SUBMISSIONS TO OUR SERVICES\",\n",
    "    # \"D. FAMILY SHARING\",\n",
    "    # \"E. SERIES PASS AND MULTI-PASS\",\n",
    "    # \"F. ADDITIONAL APP STORE TERMS (EXCLUDING APPLE ARCADE APPS)\",\n",
    "    \"G. ADDITIONAL TERMS FOR CONTENT ACQUIRED FROM THIRD PARTIES\",\n",
    "    # \"H. ADDITIONAL APPLE MUSIC TERMS\",\n",
    "    # \"I. ADDITIONAL APPLE FITNESS+ TERMS\",\n",
    "    # \"J. CARRIER MEMBERSHIP\",\n",
    "    # \"K. MISCELLANEOUS TERMS APPLICABLE TO ALL SERVICES\",\n",
    "]\n",
    "# I've manually split these into lengths of around 2500 - 3000 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A. INTRODUCTION TO OUR SERVICES',\n",
       " 'C. YOUR SUBMISSIONS TO OUR SERVICES',\n",
       " 'G. ADDITIONAL TERMS FOR CONTENT ACQUIRED FROM THIRD PARTIES']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delimiters_2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_section_23 = string_between_substrings(\n",
    "    raw_text_2023, delimiters_2023[0], delimiters_2023[1]\n",
    ")\n",
    "\n",
    "b_section_23 = string_between_substrings(\n",
    "    raw_text_2023, delimiters_2023[1], delimiters_2023[2]\n",
    ")\n",
    "\n",
    "c_section_23 = string_between_substrings(raw_text_2023, delimiters_2023[2], None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A Section length: 2654 words,\n",
      "      B Section length: 2996 words,\n",
      "      C Section length: 2692 words \n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    f\"\"\"A Section length: {len(a_section_23.split())} words,\n",
    "      B Section length: {len(b_section_23.split())} words,\n",
    "      C Section length: {len(c_section_23.split())} words \"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_doc_2023 = text_splitter.create_documents([a_section_23])\n",
    "b_doc_2023 = text_splitter.create_documents([b_section_23])\n",
    "c_doc_2023 = text_splitter.create_documents([c_section_23])\n",
    "\n",
    "a_split_2023 = text_splitter.split_documents(a_doc_2023)\n",
    "b_split_2023 = text_splitter.split_documents(b_doc_2023)\n",
    "c_split_2023 = text_splitter.split_documents(c_doc_2023)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_SECTIONS_MAP_REDUCE:\n",
    "    output_map_reduce_23_a = map_reduce_chain.run(a_split_2023)\n",
    "    output_map_reduce_23_b = map_reduce_chain.run(b_split_2023)\n",
    "    output_map_reduce_23_c = map_reduce_chain.run(c_split_2023)\n",
    "\n",
    "    sections_2023_summary = (\n",
    "        output_map_reduce_23_a\n",
    "        + \"\\n\"\n",
    "        + output_map_reduce_23_b\n",
    "        + \"\\n\"\n",
    "        + output_map_reduce_23_c\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_SECTIONS_MAP_REDUCE and SAVE_SECTIONS_MAP_REDUCE:\n",
    "    with open(Path(SUMMARY_SAVE_DIR, \"2023_sections_summary.txt\"), \"w\") as f:\n",
    "        f.write(sections_2023_summary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectors\n",
    "\n",
    "Interesting approach to summarizing large documents, inspired from this blog post:\n",
    "\n",
    "https://pashpashpash.substack.com/p/tackling-the-challenge-of-document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarize_with_vectors(\n",
    "    model_name: str,\n",
    "    document_split: list[Document],\n",
    "    map_prompt: str,\n",
    "    reduce_prompt: str,\n",
    "    num_clusters: int,\n",
    "    save_dir: Optional[Union[Path, str]] = None,\n",
    "):\n",
    "    llm = ChatOpenAI(model_name=model_name)\n",
    "    embeddings = OpenAIEmbeddings()\n",
    "    vectors = np.array(\n",
    "        embeddings.embed_documents([x.page_content for x in document_split])\n",
    "    )\n",
    "    kmeans = KMeans(n_clusters=num_clusters, random_state=42, n_init=\"auto\").fit(\n",
    "        vectors\n",
    "    )\n",
    "    # Find the closest embeddings to the centroids\n",
    "\n",
    "    closest_indices = []\n",
    "\n",
    "    for i in range(num_clusters):\n",
    "        # Get the list of distances from that particular cluster center\n",
    "        distances = np.linalg.norm(vectors - kmeans.cluster_centers_[i], axis=1)\n",
    "\n",
    "        # Find the list position of the closest one (using argmin to find the smallest distance)\n",
    "        closest_index = np.argmin(distances)\n",
    "\n",
    "        # Append that position to your closest indices list\n",
    "        closest_indices.append(closest_index)\n",
    "    selected_indices = sorted(closest_indices)\n",
    "    map_chain = LLMChain(llm=llm, prompt=map_prompt)\n",
    "    reduce_chain = LLMChain(llm=llm, prompt=reduce_prompt)\n",
    "    selected_docs = [document_split[doc] for doc in selected_indices]\n",
    "    # Make an empty list to hold your summaries\n",
    "    summary_list = []\n",
    "\n",
    "    # Loop through a range of the lenght of your selected docs\n",
    "    for i, doc in enumerate(selected_docs):\n",
    "        # Go get a summary of the chunk\n",
    "        chunk_summary = map_chain.run([doc])\n",
    "\n",
    "        # Append that summary to your list\n",
    "        summary_list.append(chunk_summary)\n",
    "\n",
    "    summaries = \"\\n\".join(summary_list)\n",
    "\n",
    "    # Convert it back to a document\n",
    "    summaries = Document(page_content=summaries)\n",
    "    output = reduce_chain.run([summaries])\n",
    "    if save_dir is not None:\n",
    "        with open(save_dir, \"w\") as f:\n",
    "            f.write(output)\n",
    "        return output\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_tsne(\n",
    "    document_split: list[Document],\n",
    "    num_clusters: int,\n",
    "    perplexity: int,\n",
    "    plot_kwargs: dict,\n",
    "    save_dir: Optional[Union[Path, str]] = None,\n",
    "):\n",
    "    colours = [\n",
    "        \"maroon\",\n",
    "        \"purple\",\n",
    "        \"green\",\n",
    "        \"blue\",\n",
    "        \"yellow\",\n",
    "        \"black\",\n",
    "        \"aqua\",\n",
    "        \"coral\",\n",
    "        \"darkblue\",\n",
    "        \"darkgreen\",\n",
    "        \"darkmagenta\",\n",
    "        \"darkslateblue\",\n",
    "        \"deeppink\",\n",
    "        \"dimgrey\",\n",
    "        \"indianred\",\n",
    "        \"mediumslateblue\",\n",
    "        \"olivedrab\",\n",
    "        \"orangered\",\n",
    "        \"palevioletred\",\n",
    "    ]\n",
    "    colour_map = {k: v for k, v in enumerate(colours)}\n",
    "\n",
    "    embeddings = OpenAIEmbeddings()\n",
    "    vectors = np.array(\n",
    "        embeddings.embed_documents([x.page_content for x in document_split])\n",
    "    )\n",
    "    tsne = TSNE(n_components=2, perplexity=perplexity, random_state=42)\n",
    "    reduced_data_tsne = tsne.fit_transform(vectors)\n",
    "    kmeans = KMeans(n_clusters=num_clusters, random_state=42, n_init=\"auto\").fit(\n",
    "        vectors\n",
    "    )\n",
    "    kmeans_colours = []\n",
    "    for label in kmeans.labels_:\n",
    "        kmeans_colours.append(colour_map[label])\n",
    "    fig = go.Figure(\n",
    "        data=go.Scatter(\n",
    "            x=reduced_data_tsne[:, 0],\n",
    "            y=reduced_data_tsne[:, 1],\n",
    "            marker_color=kmeans_colours,\n",
    "            hovertext=kmeans.labels_,\n",
    "            mode=\"markers\",\n",
    "            name=\"TSNE Cluster\",\n",
    "        )\n",
    "    )\n",
    "    fig.update_layout(\n",
    "        template=\"simple_white\",\n",
    "        title=plot_kwargs[\"title\"],\n",
    "        xaxis_title=\"TSNE 1\",\n",
    "        yaxis_title=\"TSNE 2\",\n",
    "    )\n",
    "    if save_dir is not None:\n",
    "        if save_dir.suffix != \"\":\n",
    "            save_dir = Path(\n",
    "                save_dir.parent, save_dir.stem\n",
    "            )  # Removing suffix if existing\n",
    "        save_html = str(save_dir) + \".html\"\n",
    "        save_png = str(save_dir) + \".png\"\n",
    "        with open(save_html, \"w\") as f:\n",
    "            f.write(fig.to_html())\n",
    "        fig.write_image(save_png)\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2015"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
    "    chunk_size=750, chunk_overlap=50\n",
    ")\n",
    "split_2015 = text_splitter.split_documents(data_2015)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    }
   ],
   "source": [
    "print(len(split_2015))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertext": [
          7,
          8,
          4,
          4,
          4,
          7,
          6,
          6,
          6,
          9,
          0,
          0,
          0,
          5,
          5,
          7,
          2,
          2,
          1,
          1,
          5,
          5,
          2,
          2,
          7,
          7,
          3,
          3,
          3,
          7
         ],
         "marker": {
          "color": [
           "coral",
           "darkblue",
           "yellow",
           "yellow",
           "yellow",
           "coral",
           "aqua",
           "aqua",
           "aqua",
           "darkgreen",
           "maroon",
           "maroon",
           "maroon",
           "black",
           "black",
           "coral",
           "green",
           "green",
           "purple",
           "purple",
           "black",
           "black",
           "green",
           "green",
           "coral",
           "coral",
           "blue",
           "blue",
           "blue",
           "coral"
          ]
         },
         "mode": "markers",
         "name": "TSNE Cluster",
         "type": "scatter",
         "x": [
          0.5919467210769653,
          0.4922710657119751,
          1.59365975856781,
          1.9333815574645996,
          1.2453625202178955,
          1.4879509210586548,
          2.3494460582733154,
          2.352928400039673,
          1.7943776845932007,
          2.088825225830078,
          -1.3772379159927368,
          -1.159244179725647,
          -0.8546536564826965,
          -0.5226016640663147,
          0.08673490583896637,
          0.6039941906929016,
          1.8437715768814087,
          0.8052049279212952,
          -0.7021022439002991,
          -0.5527239441871643,
          -0.40449613332748413,
          0.2896997630596161,
          0.6696463227272034,
          1.186274528503418,
          0.4042304754257202,
          -0.007422410883009434,
          -1.2758151292800903,
          -1.30910325050354,
          -1.3938281536102295,
          -0.11113469302654266
         ],
         "y": [
          -4.377821922302246,
          -5.121522903442383,
          -4.456004619598389,
          -4.934879302978516,
          -4.335800647735596,
          -3.5986087322235107,
          -3.1477086544036865,
          -2.589970827102661,
          -2.4050817489624023,
          -3.865748882293701,
          -3.461541175842285,
          -3.30915904045105,
          -3.867372512817383,
          -4.451516151428223,
          -3.991523027420044,
          -3.550978183746338,
          -3.295239210128784,
          -3.0580854415893555,
          -3.324805974960327,
          -3.7951035499572754,
          -4.367926120758057,
          -3.463583469390869,
          -2.443814754486084,
          -1.8761403560638428,
          -2.5503461360931396,
          -2.262704610824585,
          -1.7615580558776855,
          -1.77755606174469,
          -1.7743501663208008,
          -2.67110013961792
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "rgb(36,36,36)"
            },
            "error_y": {
             "color": "rgb(36,36,36)"
            },
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "baxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.6
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "rgb(237,237,237)"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "rgb(217,217,217)"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 1,
            "tickcolor": "rgb(36,36,36)",
            "ticks": "outside"
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "rgb(103,0,31)"
            ],
            [
             0.1,
             "rgb(178,24,43)"
            ],
            [
             0.2,
             "rgb(214,96,77)"
            ],
            [
             0.3,
             "rgb(244,165,130)"
            ],
            [
             0.4,
             "rgb(253,219,199)"
            ],
            [
             0.5,
             "rgb(247,247,247)"
            ],
            [
             0.6,
             "rgb(209,229,240)"
            ],
            [
             0.7,
             "rgb(146,197,222)"
            ],
            [
             0.8,
             "rgb(67,147,195)"
            ],
            [
             0.9,
             "rgb(33,102,172)"
            ],
            [
             1,
             "rgb(5,48,97)"
            ]
           ],
           "sequential": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ]
          },
          "colorway": [
           "#1F77B4",
           "#FF7F0E",
           "#2CA02C",
           "#D62728",
           "#9467BD",
           "#8C564B",
           "#E377C2",
           "#7F7F7F",
           "#BCBD22",
           "#17BECF"
          ],
          "font": {
           "color": "rgb(36,36,36)"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "white",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "white",
          "polar": {
           "angularaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "radialaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "yaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "zaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           }
          },
          "shapedefaults": {
           "fillcolor": "black",
           "line": {
            "width": 0
           },
           "opacity": 0.3
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "baxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "caxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          }
         }
        },
        "title": {
         "text": "2015 Contract TSNE Plot"
        },
        "xaxis": {
         "title": {
          "text": "TSNE 1"
         }
        },
        "yaxis": {
         "title": {
          "text": "TSNE 2"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne_2015_fname = Path(\"figures\", \"tsne_2015_fig.html\")\n",
    "plot_kwargs_2015 = {\"title\": \"2015 Contract TSNE Plot\"}\n",
    "plot_tsne(\n",
    "    split_2015,\n",
    "    num_clusters=10,\n",
    "    perplexity=20,\n",
    "    plot_kwargs=plot_kwargs_2015,\n",
    "    save_dir=tsne_2015_fname,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_2015_summary_fname = Path(\"data\", \"summaries\", \"vector_2015_summary.txt\")\n",
    "if not vector_2015_summary_fname.exists():\n",
    "    summarize_with_vectors(\n",
    "        model_name=\"gpt-3.5-turbo\",\n",
    "        document_split=split_2015,\n",
    "        map_prompt=map_prompt,\n",
    "        reduce_prompt=reduce_prompt,\n",
    "        num_clusters=5,\n",
    "        save_dir=vector_2015_summary_fname,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_2023 = text_splitter.split_documents(data_2023)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    }
   ],
   "source": [
    "print(len(split_2023))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertext": [
          2,
          8,
          2,
          4,
          4,
          4,
          0,
          6,
          5,
          1,
          1,
          2,
          2,
          7,
          9,
          1,
          3
         ],
         "marker": {
          "color": [
           "green",
           "darkblue",
           "green",
           "yellow",
           "yellow",
           "yellow",
           "maroon",
           "aqua",
           "black",
           "purple",
           "purple",
           "green",
           "green",
           "coral",
           "darkgreen",
           "purple",
           "blue"
          ]
         },
         "mode": "markers",
         "name": "TSNE Cluster",
         "type": "scatter",
         "x": [
          -15.135077476501465,
          35.74640655517578,
          7.197092533111572,
          -38.75954818725586,
          -52.0947151184082,
          -29.1043758392334,
          11.159340858459473,
          -17.23189353942871,
          8.864479064941406,
          36.96124267578125,
          40.42979049682617,
          7.068820476531982,
          -39.40673065185547,
          -3.171734094619751,
          -15.916979789733887,
          21.046939849853516,
          -18.009660720825195
         ],
         "y": [
          -17.43739891052246,
          30.111433029174805,
          -17.563644409179688,
          -27.309911727905273,
          -5.665297985076904,
          -0.1624424010515213,
          53.81703567504883,
          -44.163169860839844,
          -45.13117980957031,
          -31.335073471069336,
          -6.274085521697998,
          26.129737854003906,
          22.304880142211914,
          2.7789740562438965,
          19.79369354248047,
          6.262296199798584,
          43.67189407348633
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "rgb(36,36,36)"
            },
            "error_y": {
             "color": "rgb(36,36,36)"
            },
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "baxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.6
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "rgb(237,237,237)"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "rgb(217,217,217)"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 1,
            "tickcolor": "rgb(36,36,36)",
            "ticks": "outside"
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "rgb(103,0,31)"
            ],
            [
             0.1,
             "rgb(178,24,43)"
            ],
            [
             0.2,
             "rgb(214,96,77)"
            ],
            [
             0.3,
             "rgb(244,165,130)"
            ],
            [
             0.4,
             "rgb(253,219,199)"
            ],
            [
             0.5,
             "rgb(247,247,247)"
            ],
            [
             0.6,
             "rgb(209,229,240)"
            ],
            [
             0.7,
             "rgb(146,197,222)"
            ],
            [
             0.8,
             "rgb(67,147,195)"
            ],
            [
             0.9,
             "rgb(33,102,172)"
            ],
            [
             1,
             "rgb(5,48,97)"
            ]
           ],
           "sequential": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ]
          },
          "colorway": [
           "#1F77B4",
           "#FF7F0E",
           "#2CA02C",
           "#D62728",
           "#9467BD",
           "#8C564B",
           "#E377C2",
           "#7F7F7F",
           "#BCBD22",
           "#17BECF"
          ],
          "font": {
           "color": "rgb(36,36,36)"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "white",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "white",
          "polar": {
           "angularaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "radialaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "yaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "zaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           }
          },
          "shapedefaults": {
           "fillcolor": "black",
           "line": {
            "width": 0
           },
           "opacity": 0.3
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "baxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "caxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          }
         }
        },
        "title": {
         "text": "2023 Contract TSNE Plot"
        },
        "xaxis": {
         "title": {
          "text": "TSNE 1"
         }
        },
        "yaxis": {
         "title": {
          "text": "TSNE 2"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne_2023_fname = Path(\"figures\", \"tsne_2023_fig.html\")\n",
    "plot_kwargs_2023 = {\"title\": \"2023 Contract TSNE Plot\"}\n",
    "plot_tsne(\n",
    "    split_2023,\n",
    "    num_clusters=10,\n",
    "    perplexity=15,\n",
    "    plot_kwargs=plot_kwargs_2023,\n",
    "    save_dir=tsne_2023_fname,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_2023_summary_fname = Path(\"data\", \"summaries\", \"vector_2023_summary.txt\")\n",
    "if not vector_2023_summary_fname.exists():\n",
    "    summarize_with_vectors(\n",
    "        model_name=\"gpt-3.5-turbo\",\n",
    "        document_split=split_2023,\n",
    "        map_prompt=map_prompt,\n",
    "        reduce_prompt=reduce_prompt,\n",
    "        num_clusters=5,\n",
    "        save_dir=vector_2023_summary_fname,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}