stanzaDependency.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "51825e42",
   "metadata": {},
   "source": [
    "### Exploring dependency tree parsing using stanza and also CoreNLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "738e4438",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-11 10:35:48.907092: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-11 10:35:49.066631: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2023-04-11 10:35:49.572226: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-11 10:35:49.572319: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-04-11 10:35:49.572326: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "2023-04-11 10:35:50.164645: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2023-04-11 10:35:50.164661: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-04-11 10:35:50.164690: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (muddy-HP-ProDesk-600-G3-SFF): /proc/driver/nvidia/version does not exist\n",
      "[nltk_data] Downloading package punkt to /home/muddy/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import unicodedata\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import warnings\n",
    "import nltk\n",
    "import spacy\n",
    "import stanza\n",
    "import en_core_web_md\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics.pairwise import euclidean_distances\n",
    "from collections import defaultdict\n",
    "from textblob import TextBlob\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from nltk.tokenize import sent_tokenize\n",
    "nltk.download('punkt')\n",
    "word_token = TreebankWordTokenizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29554c8a",
   "metadata": {},
   "source": [
    "### Load up the speeches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "51b5e575",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>filepath</th>\n",
       "      <th>date</th>\n",
       "      <th>source</th>\n",
       "      <th>sentences</th>\n",
       "      <th>words</th>\n",
       "      <th>num_sents</th>\n",
       "      <th>num_words</th>\n",
       "      <th>word_set</th>\n",
       "      <th>num_unique_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Thank you for coming. Prime Minister Olmert, P...</td>\n",
       "      <td>./Data/GWB/remarks-the-annapolis-conference-annapol...</td>\n",
       "      <td>2007-11-27</td>\n",
       "      <td>gwb</td>\n",
       "      <td>[Thank you for coming., Prime Minister Olmert,...</td>\n",
       "      <td>[Thank, you, for, coming., Prime, Minister, Ol...</td>\n",
       "      <td>116</td>\n",
       "      <td>2461</td>\n",
       "      <td>{commitment, assistance, toward, concluded, Is...</td>\n",
       "      <td>739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Good evening. During the next few minutes, I w...</td>\n",
       "      <td>./Data/GWB/address-the-nation-the-proposed-departme...</td>\n",
       "      <td>2002-06-06</td>\n",
       "      <td>gwb</td>\n",
       "      <td>[Good evening., During the next few minutes, I...</td>\n",
       "      <td>[Good, evening., During, the, next, few, minut...</td>\n",
       "      <td>71</td>\n",
       "      <td>1605</td>\n",
       "      <td>{finish, thrust, assistance, emerging, conclud...</td>\n",
       "      <td>674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Our Nation is shocked and saddened by the news...</td>\n",
       "      <td>./Data/GWB/remarks-the-shootings-virginia-tech-blac...</td>\n",
       "      <td>2007-04-16</td>\n",
       "      <td>gwb</td>\n",
       "      <td>[Our Nation is shocked and saddened by the new...</td>\n",
       "      <td>[Our, Nation, is, shocked, and, saddened, by, ...</td>\n",
       "      <td>10</td>\n",
       "      <td>207</td>\n",
       "      <td>{are, pledged, be, American, have, ones, I, Sc...</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Thank you for coming. Prime Minister Olmert, P...   \n",
       "1  Good evening. During the next few minutes, I w...   \n",
       "2  Our Nation is shocked and saddened by the news...   \n",
       "\n",
       "                                            filepath       date source  \\\n",
       "0  ./Data/GWB/remarks-the-annapolis-conference-annapol... 2007-11-27    gwb   \n",
       "1  ./Data/GWB/address-the-nation-the-proposed-departme... 2002-06-06    gwb   \n",
       "2  ./Data/GWB/remarks-the-shootings-virginia-tech-blac... 2007-04-16    gwb   \n",
       "\n",
       "                                           sentences  \\\n",
       "0  [Thank you for coming., Prime Minister Olmert,...   \n",
       "1  [Good evening., During the next few minutes, I...   \n",
       "2  [Our Nation is shocked and saddened by the new...   \n",
       "\n",
       "                                               words  num_sents  num_words  \\\n",
       "0  [Thank, you, for, coming., Prime, Minister, Ol...        116       2461   \n",
       "1  [Good, evening., During, the, next, few, minut...         71       1605   \n",
       "2  [Our, Nation, is, shocked, and, saddened, by, ...         10        207   \n",
       "\n",
       "                                            word_set  num_unique_words  \n",
       "0  {commitment, assistance, toward, concluded, Is...               739  \n",
       "1  {finish, thrust, assistance, emerging, conclud...               674  \n",
       "2  {are, pledged, be, American, have, ones, I, Sc...               128  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load up the files\n",
    "#paths = ['./Data/speeches/', './Data/NYTimes/', './Data/WSJ/'] \n",
    "paths = ['./Data/GWB/']\n",
    "list_of_files = []\n",
    "\n",
    "dates = pd.read_csv('./Data/genData/dateSpeeches.csv')\n",
    "dates = pd.read_csv('./Data/genData/speech_and_date_gwb.csv')\n",
    "for path in paths:\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file in files:\n",
    "            if file.endswith('.txt'):\n",
    "                list_of_files.append(os.path.join(root,file))\n",
    "\n",
    "speeches = []\n",
    "for file in list_of_files:\n",
    "    with open(file, encoding='utf-8') as f:\n",
    "        #print(file)\n",
    "        text = f.read()\n",
    "    f.close()\n",
    "    speeches.append([text, file])\n",
    "\n",
    "#clean out goofy unicode  space characters \n",
    "speeches = [(unicodedata.normalize(\"NFKD\", speech[0]), speech[1]) for speech in speeches if len(speech)>0 ]\n",
    "\n",
    "# remove [stuff] in between square brackets\n",
    "def remove_bracket(text):\n",
    "    return re.sub('(\\[[^w]*\\]\\s)', '',text)\n",
    "speeches = [(remove_bracket(speech[0]), speech[1]) for speech in speeches]\n",
    "\n",
    "def get_source(text):\n",
    "    regex = \"[^./][a-zA-Z]+[^/]\"\n",
    "    string = re.findall(regex, str(text))[0]\n",
    "    if string == 'speeches': string = 'oba'\n",
    "    if string == 'NYTimes': string = 'nyt'\n",
    "    return string.lower()\n",
    "\n",
    "def get_date(text):\n",
    "    regex = \"([0-9]+[\\-][0-9]+[\\-][0-9]+)\"\n",
    "    return re.findall(regex, str(text))[0]\n",
    "\n",
    "def get_filename(text):\n",
    "    regex = \"[-]([a-zA-Z]+)\"\n",
    "    return re.findall(regex, str(text))[0]\n",
    "\n",
    "cols = ['text', 'filepath']\n",
    "text_df = pd.DataFrame(speeches, columns=cols)\n",
    "# A couple tweaks for the GWB data\n",
    "dates['file'] = [ file.replace('GWB/', './Data/GWB/') for file in dates['file'] ]\n",
    "dates = dates.rename(columns={\"file\": \"filepath\"})\n",
    "#text_df['date'] = text_df['filepath'].apply(get_date)\n",
    "text_df = pd.merge(text_df, dates, how='left', on='filepath')\n",
    "text_df['date'] = pd.to_datetime(text_df['date'], format='%Y-%m-%d')\n",
    "text_df['source'] = text_df['filepath'].apply(get_source)\n",
    "\n",
    "text_df['sentences'] = text_df['text'].apply(sent_tokenize)\n",
    "text_df['words'] = text_df['text'].apply(word_token.tokenize)\n",
    "text_df['num_sents'] = text_df['sentences'].apply(len)\n",
    "text_df['num_words'] = text_df['words'].apply(len)\n",
    "text_df['word_set'] = text_df['words'].apply(set)\n",
    "text_df['num_unique_words'] = text_df['word_set'].apply(len)\n",
    "text_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06a037a1",
   "metadata": {},
   "source": [
    "<A HREF=\"https://textblob.readthedocs.io/en/latest/quickstart.html\">TextBlob Quickstart guide</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce3633e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_df['TBsubjectivity']=[TextBlob(text).sentiment.subjectivity for text in text_df['text']]\n",
    "text_df['TBpolarity']=[TextBlob(text).sentiment.polarity for text in text_df['text']]\n",
    "#text_df.to_csv('./Data/genData/numwords_TBpolar_gwb.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00a08e2b",
   "metadata": {},
   "source": [
    "<A HREF=\"https://plotly.com/python/plotly-express/\">Plotly Express</A><BR><A HREF=\"https://pandas.pydata.org/Pandas_Cheat_Sheet.pdf\">Pandas cheat sheet</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83cda881",
   "metadata": {},
   "source": [
    "<A HREF=\"https://universaldependencies.org/u/pos/\">Universal POS tags</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dbcd107",
   "metadata": {},
   "source": [
    "<A HREF=\"https://en.wikipedia.org/wiki/Interjection\">Wikipedia - Interjections</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec3a64d1",
   "metadata": {},
   "source": [
    "<A HREF=\"https://pypi.org/project/NRCLex/\">NRCLex</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ce01fae",
   "metadata": {},
   "source": [
    "### Attempt 1. Stanza"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ca14c82",
   "metadata": {},
   "source": [
    "### <A HREF=\"https://stanfordnlp.github.io/stanza/getting_started.html\">Stanza quickstart guide</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ac01a37",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-11 10:35:52 INFO: Checking for updates to resources.json in case models have been updated.  Note: this behavior can be turned off with download_method=None or download_method=DownloadMethod.REUSE_RESOURCES\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b3a772a1a1d4c13be1f3c12cbdc16fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.5.0.json:   0%|   …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-11 10:35:54 INFO: Loading these models for language: en (English):\n",
      "============================\n",
      "| Processor    | Package   |\n",
      "----------------------------\n",
      "| tokenize     | combined  |\n",
      "| pos          | combined  |\n",
      "| lemma        | combined  |\n",
      "| constituency | wsj       |\n",
      "| depparse     | combined  |\n",
      "| sentiment    | sstplus   |\n",
      "| ner          | ontonotes |\n",
      "============================\n",
      "\n",
      "2023-04-11 10:35:54 INFO: Using device: cpu\n",
      "2023-04-11 10:35:54 INFO: Loading: tokenize\n",
      "2023-04-11 10:35:54 INFO: Loading: pos\n",
      "2023-04-11 10:35:54 INFO: Loading: lemma\n",
      "2023-04-11 10:35:54 INFO: Loading: constituency\n",
      "2023-04-11 10:35:55 INFO: Loading: depparse\n",
      "2023-04-11 10:35:55 INFO: Loading: sentiment\n",
      "2023-04-11 10:35:55 INFO: Loading: ner\n",
      "2023-04-11 10:35:56 INFO: Done loading processors!\n"
     ]
    }
   ],
   "source": [
    "nlp = stanza.Pipeline(lang=\"en\") # Initialize the default English pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9c295d30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\n",
       "  [\n",
       "    {\n",
       "      \"id\": 1,\n",
       "      \"text\": \"With\",\n",
       "      \"lemma\": \"with\",\n",
       "      \"upos\": \"ADP\",\n",
       "      \"xpos\": \"IN\",\n",
       "      \"head\": 5,\n",
       "      \"deprel\": \"case\",\n",
       "      \"start_char\": 0,\n",
       "      \"end_char\": 4,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 2,\n",
       "      \"text\": \"Al\",\n",
       "      \"lemma\": \"Al\",\n",
       "      \"upos\": \"PROPN\",\n",
       "      \"xpos\": \"NNP\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 5,\n",
       "      \"deprel\": \"nmod:poss\",\n",
       "      \"start_char\": 5,\n",
       "      \"end_char\": 7,\n",
       "      \"ner\": \"S-PERSON\",\n",
       "      \"multi_ner\": [\n",
       "        \"S-PERSON\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 3,\n",
       "      \"text\": \"'s\",\n",
       "      \"lemma\": \"'s\",\n",
       "      \"upos\": \"PART\",\n",
       "      \"xpos\": \"POS\",\n",
       "      \"head\": 2,\n",
       "      \"deprel\": \"case\",\n",
       "      \"start_char\": 7,\n",
       "      \"end_char\": 9,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 4,\n",
       "      \"text\": \"principled\",\n",
       "      \"lemma\": \"principled\",\n",
       "      \"upos\": \"ADJ\",\n",
       "      \"xpos\": \"JJ\",\n",
       "      \"feats\": \"Degree=Pos\",\n",
       "      \"head\": 5,\n",
       "      \"deprel\": \"amod\",\n",
       "      \"start_char\": 10,\n",
       "      \"end_char\": 20,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 5,\n",
       "      \"text\": \"leadership\",\n",
       "      \"lemma\": \"leadership\",\n",
       "      \"upos\": \"NOUN\",\n",
       "      \"xpos\": \"NN\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"obl\",\n",
       "      \"start_char\": 21,\n",
       "      \"end_char\": 31,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 6,\n",
       "      \"text\": \",\",\n",
       "      \"lemma\": \",\",\n",
       "      \"upos\": \"PUNCT\",\n",
       "      \"xpos\": \",\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"punct\",\n",
       "      \"start_char\": 31,\n",
       "      \"end_char\": 32,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 7,\n",
       "      \"text\": \"the\",\n",
       "      \"lemma\": \"the\",\n",
       "      \"upos\": \"DET\",\n",
       "      \"xpos\": \"DT\",\n",
       "      \"feats\": \"Definite=Def|PronType=Art\",\n",
       "      \"head\": 8,\n",
       "      \"deprel\": \"det\",\n",
       "      \"start_char\": 33,\n",
       "      \"end_char\": 36,\n",
       "      \"ner\": \"B-ORG\",\n",
       "      \"multi_ner\": [\n",
       "        \"B-ORG\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 8,\n",
       "      \"text\": \"Department\",\n",
       "      \"lemma\": \"Department\",\n",
       "      \"upos\": \"PROPN\",\n",
       "      \"xpos\": \"NNP\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"nsubj\",\n",
       "      \"start_char\": 37,\n",
       "      \"end_char\": 47,\n",
       "      \"ner\": \"I-ORG\",\n",
       "      \"multi_ner\": [\n",
       "        \"I-ORG\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 9,\n",
       "      \"text\": \"of\",\n",
       "      \"lemma\": \"of\",\n",
       "      \"upos\": \"ADP\",\n",
       "      \"xpos\": \"IN\",\n",
       "      \"head\": 10,\n",
       "      \"deprel\": \"case\",\n",
       "      \"start_char\": 48,\n",
       "      \"end_char\": 50,\n",
       "      \"ner\": \"I-ORG\",\n",
       "      \"multi_ner\": [\n",
       "        \"I-ORG\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 10,\n",
       "      \"text\": \"Justice\",\n",
       "      \"lemma\": \"Justice\",\n",
       "      \"upos\": \"PROPN\",\n",
       "      \"xpos\": \"NNP\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 8,\n",
       "      \"deprel\": \"nmod\",\n",
       "      \"start_char\": 51,\n",
       "      \"end_char\": 58,\n",
       "      \"ner\": \"E-ORG\",\n",
       "      \"multi_ner\": [\n",
       "        \"E-ORG\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 11,\n",
       "      \"text\": \"will\",\n",
       "      \"lemma\": \"will\",\n",
       "      \"upos\": \"AUX\",\n",
       "      \"xpos\": \"MD\",\n",
       "      \"feats\": \"VerbForm=Fin\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"aux\",\n",
       "      \"start_char\": 59,\n",
       "      \"end_char\": 63,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 12,\n",
       "      \"text\": \"continue\",\n",
       "      \"lemma\": \"continue\",\n",
       "      \"upos\": \"VERB\",\n",
       "      \"xpos\": \"VB\",\n",
       "      \"feats\": \"VerbForm=Inf\",\n",
       "      \"head\": 0,\n",
       "      \"deprel\": \"root\",\n",
       "      \"start_char\": 64,\n",
       "      \"end_char\": 72,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 13,\n",
       "      \"text\": \"this\",\n",
       "      \"lemma\": \"this\",\n",
       "      \"upos\": \"DET\",\n",
       "      \"xpos\": \"DT\",\n",
       "      \"feats\": \"Number=Sing|PronType=Dem\",\n",
       "      \"head\": 15,\n",
       "      \"deprel\": \"det\",\n",
       "      \"start_char\": 73,\n",
       "      \"end_char\": 77,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 14,\n",
       "      \"text\": \"important\",\n",
       "      \"lemma\": \"important\",\n",
       "      \"upos\": \"ADJ\",\n",
       "      \"xpos\": \"JJ\",\n",
       "      \"feats\": \"Degree=Pos\",\n",
       "      \"head\": 15,\n",
       "      \"deprel\": \"amod\",\n",
       "      \"start_char\": 78,\n",
       "      \"end_char\": 87,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 15,\n",
       "      \"text\": \"mission\",\n",
       "      \"lemma\": \"mission\",\n",
       "      \"upos\": \"NOUN\",\n",
       "      \"xpos\": \"NN\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"obj\",\n",
       "      \"start_char\": 88,\n",
       "      \"end_char\": 95,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 16,\n",
       "      \"text\": \"and\",\n",
       "      \"lemma\": \"and\",\n",
       "      \"upos\": \"CCONJ\",\n",
       "      \"xpos\": \"CC\",\n",
       "      \"head\": 18,\n",
       "      \"deprel\": \"cc\",\n",
       "      \"start_char\": 96,\n",
       "      \"end_char\": 99,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 17,\n",
       "      \"text\": \"will\",\n",
       "      \"lemma\": \"will\",\n",
       "      \"upos\": \"AUX\",\n",
       "      \"xpos\": \"MD\",\n",
       "      \"feats\": \"VerbForm=Fin\",\n",
       "      \"head\": 18,\n",
       "      \"deprel\": \"aux\",\n",
       "      \"start_char\": 100,\n",
       "      \"end_char\": 104,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 18,\n",
       "      \"text\": \"defend\",\n",
       "      \"lemma\": \"defend\",\n",
       "      \"upos\": \"VERB\",\n",
       "      \"xpos\": \"VB\",\n",
       "      \"feats\": \"VerbForm=Inf\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"conj\",\n",
       "      \"start_char\": 105,\n",
       "      \"end_char\": 111,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 19,\n",
       "      \"text\": \"the\",\n",
       "      \"lemma\": \"the\",\n",
       "      \"upos\": \"DET\",\n",
       "      \"xpos\": \"DT\",\n",
       "      \"feats\": \"Definite=Def|PronType=Art\",\n",
       "      \"head\": 20,\n",
       "      \"deprel\": \"det\",\n",
       "      \"start_char\": 112,\n",
       "      \"end_char\": 115,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 20,\n",
       "      \"text\": \"security\",\n",
       "      \"lemma\": \"security\",\n",
       "      \"upos\": \"NOUN\",\n",
       "      \"xpos\": \"NN\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 18,\n",
       "      \"deprel\": \"obj\",\n",
       "      \"start_char\": 116,\n",
       "      \"end_char\": 124,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 21,\n",
       "      \"text\": \"of\",\n",
       "      \"lemma\": \"of\",\n",
       "      \"upos\": \"ADP\",\n",
       "      \"xpos\": \"IN\",\n",
       "      \"head\": 23,\n",
       "      \"deprel\": \"case\",\n",
       "      \"start_char\": 125,\n",
       "      \"end_char\": 127,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 22,\n",
       "      \"text\": \"all\",\n",
       "      \"lemma\": \"all\",\n",
       "      \"upos\": \"DET\",\n",
       "      \"xpos\": \"DT\",\n",
       "      \"head\": 23,\n",
       "      \"deprel\": \"det\",\n",
       "      \"start_char\": 128,\n",
       "      \"end_char\": 131,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 23,\n",
       "      \"text\": \"Americans\",\n",
       "      \"lemma\": \"American\",\n",
       "      \"upos\": \"PROPN\",\n",
       "      \"xpos\": \"NNPS\",\n",
       "      \"feats\": \"Number=Plur\",\n",
       "      \"head\": 20,\n",
       "      \"deprel\": \"nmod\",\n",
       "      \"start_char\": 132,\n",
       "      \"end_char\": 141,\n",
       "      \"ner\": \"S-NORP\",\n",
       "      \"multi_ner\": [\n",
       "        \"S-NORP\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 24,\n",
       "      \"text\": \"and\",\n",
       "      \"lemma\": \"and\",\n",
       "      \"upos\": \"CCONJ\",\n",
       "      \"xpos\": \"CC\",\n",
       "      \"head\": 26,\n",
       "      \"deprel\": \"cc\",\n",
       "      \"start_char\": 142,\n",
       "      \"end_char\": 145,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 25,\n",
       "      \"text\": \"the\",\n",
       "      \"lemma\": \"the\",\n",
       "      \"upos\": \"DET\",\n",
       "      \"xpos\": \"DT\",\n",
       "      \"feats\": \"Definite=Def|PronType=Art\",\n",
       "      \"head\": 26,\n",
       "      \"deprel\": \"det\",\n",
       "      \"start_char\": 146,\n",
       "      \"end_char\": 149,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 26,\n",
       "      \"text\": \"liberty\",\n",
       "      \"lemma\": \"liberty\",\n",
       "      \"upos\": \"NOUN\",\n",
       "      \"xpos\": \"NN\",\n",
       "      \"feats\": \"Number=Sing\",\n",
       "      \"head\": 20,\n",
       "      \"deprel\": \"conj\",\n",
       "      \"start_char\": 150,\n",
       "      \"end_char\": 157,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 27,\n",
       "      \"text\": \"of\",\n",
       "      \"lemma\": \"of\",\n",
       "      \"upos\": \"ADP\",\n",
       "      \"xpos\": \"IN\",\n",
       "      \"head\": 29,\n",
       "      \"deprel\": \"case\",\n",
       "      \"start_char\": 158,\n",
       "      \"end_char\": 160,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 28,\n",
       "      \"text\": \"all\",\n",
       "      \"lemma\": \"all\",\n",
       "      \"upos\": \"DET\",\n",
       "      \"xpos\": \"DT\",\n",
       "      \"head\": 29,\n",
       "      \"deprel\": \"det\",\n",
       "      \"start_char\": 161,\n",
       "      \"end_char\": 164,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 29,\n",
       "      \"text\": \"Americans\",\n",
       "      \"lemma\": \"American\",\n",
       "      \"upos\": \"PROPN\",\n",
       "      \"xpos\": \"NNPS\",\n",
       "      \"feats\": \"Number=Plur\",\n",
       "      \"head\": 26,\n",
       "      \"deprel\": \"nmod\",\n",
       "      \"start_char\": 165,\n",
       "      \"end_char\": 174,\n",
       "      \"ner\": \"S-NORP\",\n",
       "      \"multi_ner\": [\n",
       "        \"S-NORP\"\n",
       "      ]\n",
       "    },\n",
       "    {\n",
       "      \"id\": 30,\n",
       "      \"text\": \".\",\n",
       "      \"lemma\": \".\",\n",
       "      \"upos\": \"PUNCT\",\n",
       "      \"xpos\": \".\",\n",
       "      \"head\": 12,\n",
       "      \"deprel\": \"punct\",\n",
       "      \"start_char\": 174,\n",
       "      \"end_char\": 175,\n",
       "      \"ner\": \"O\",\n",
       "      \"multi_ner\": [\n",
       "        \"O\"\n",
       "      ]\n",
       "    }\n",
       "  ]\n",
       "]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "intxt=stanza.Document([], text=[sentence for sentence in text_df['sentences']][20][20])\n",
    "out=nlp(intxt)\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3db8bdf1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id: 1\tword: With\thead id: 5\thead: leadership\tdeprel: case\n",
      "id: 2\tword: Al\thead id: 5\thead: leadership\tdeprel: nmod:poss\n",
      "id: 3\tword: 's\thead id: 2\thead: Al\tdeprel: case\n",
      "id: 4\tword: principled\thead id: 5\thead: leadership\tdeprel: amod\n",
      "id: 5\tword: leadership\thead id: 12\thead: continue\tdeprel: obl\n",
      "id: 6\tword: ,\thead id: 12\thead: continue\tdeprel: punct\n",
      "id: 7\tword: the\thead id: 8\thead: Department\tdeprel: det\n",
      "id: 8\tword: Department\thead id: 12\thead: continue\tdeprel: nsubj\n",
      "id: 9\tword: of\thead id: 10\thead: Justice\tdeprel: case\n",
      "id: 10\tword: Justice\thead id: 8\thead: Department\tdeprel: nmod\n",
      "id: 11\tword: will\thead id: 12\thead: continue\tdeprel: aux\n",
      "id: 12\tword: continue\thead id: 0\thead: root\tdeprel: root\n",
      "id: 13\tword: this\thead id: 15\thead: mission\tdeprel: det\n",
      "id: 14\tword: important\thead id: 15\thead: mission\tdeprel: amod\n",
      "id: 15\tword: mission\thead id: 12\thead: continue\tdeprel: obj\n",
      "id: 16\tword: and\thead id: 18\thead: defend\tdeprel: cc\n",
      "id: 17\tword: will\thead id: 18\thead: defend\tdeprel: aux\n",
      "id: 18\tword: defend\thead id: 12\thead: continue\tdeprel: conj\n",
      "id: 19\tword: the\thead id: 20\thead: security\tdeprel: det\n",
      "id: 20\tword: security\thead id: 18\thead: defend\tdeprel: obj\n",
      "id: 21\tword: of\thead id: 23\thead: Americans\tdeprel: case\n",
      "id: 22\tword: all\thead id: 23\thead: Americans\tdeprel: det\n",
      "id: 23\tword: Americans\thead id: 20\thead: security\tdeprel: nmod\n",
      "id: 24\tword: and\thead id: 26\thead: liberty\tdeprel: cc\n",
      "id: 25\tword: the\thead id: 26\thead: liberty\tdeprel: det\n",
      "id: 26\tword: liberty\thead id: 20\thead: security\tdeprel: conj\n",
      "id: 27\tword: of\thead id: 29\thead: Americans\tdeprel: case\n",
      "id: 28\tword: all\thead id: 29\thead: Americans\tdeprel: det\n",
      "id: 29\tword: Americans\thead id: 26\thead: liberty\tdeprel: nmod\n",
      "id: 30\tword: .\thead id: 12\thead: continue\tdeprel: punct\n"
     ]
    }
   ],
   "source": [
    "print(*[f'id: {word.id}\\tword: {word.text}\\thead id: {word.head}\\thead: {sent.words[word.head-1].text if word.head > 0 else \"root\"}\\tdeprel: {word.deprel}' for sent in out.sentences for word in sent.words], sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab29dc83",
   "metadata": {},
   "source": [
    "### Attempt 2. Stanford's CoreNLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1fbe0726",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using Stanford's CoreNLP parser with NLTK\n",
    "# 1. Download CoreNLP from https://stanfordnlp.github.io/CoreNLP/download.html\n",
    "# 2. make sure Java is installed, otherwise download and install Java - https://www.java.com/en/download/windows_manual.jsp\n",
    "# 3. Unzip/extract CoreNLP zip file to a directory\n",
    "# 4. Go to that directory and open a command terminal, and run the following command...\n",
    "# 4b. on my laptop its in C:\\Users\\peter\\stanford-corenlp-4.5.2\n",
    "# 5. java -mx4g -cp \"*\" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port 9000 -timeout 15000\n",
    "# 6. Now for graphviz if you want to view the parse trees, download from https://graphviz.org/download/ then install\n",
    "# 7. Now, can run the following python code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "39a8c983",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error: Could not find or load main class edu.stanford.nlp.pipeline.StanfordCoreNLPServer\r\n",
      "Caused by: java.lang.ClassNotFoundException: edu.stanford.nlp.pipeline.StanfordCoreNLPServer\r\n"
     ]
    }
   ],
   "source": [
    "# Can try this here, but may have to run from cmd terminal before continuing\n",
    "!java -mx4g -cp \"*\" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -port 9000 -timeout 15000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "973ce316",
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Source\n",
    "from nltk.parse.corenlp import CoreNLPDependencyParser\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "15f327ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"PATH\"] += os.pathsep + 'C:/Program Files/Graphviz/bin/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09006e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdp = CoreNLPDependencyParser()\n",
    "sentence = [sentence for sentence in text_df['sentences']][20][20]\n",
    "result = list(sdp.raw_parse(sentence))\n",
    "dep_tree_dot_repr = [parse for parse in result][0].to_dot()\n",
    "source = Source(dep_tree_dot_repr, filename=\"dep_tree\", format='png')\n",
    "source.view()\n",
    "# Opens in pop-under window... well isn't that nice!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80379c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Graph image doesn't get saved, need to re-run the code\n",
    "source"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b09391d",
   "metadata": {},
   "source": [
    "### Attempt 3. Spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7267ec6e",
   "metadata": {},
   "source": [
    "<A HREF=\"https://stackoverflow.com/questions/64591644/how-to-get-height-of-dependency-tree-with-spacy\">Followed this code</A> couldn't have done it without this!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9c10ffc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'continue': 0, 'With': 1, 'leadership': 2, 'Al': 3, \"'s\": 4, 'principled': 3, ',': 1, 'Department': 1, 'the': 6, 'of': 6, 'Justice': 3, 'will': 2, 'mission': 1, 'this': 2, 'important': 2, 'and': 5, 'defend': 1, 'security': 2, 'Americans': 7, 'all': 8, 'liberty': 5, '.': 1}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# Try this just to the the height of the parse tree... using spacy\n",
    "nlp = spacy.load(\"en_core_web_md\")\n",
    "doc = nlp(sentence)\n",
    "depths = {}\n",
    "def walk_tree(node, depth):\n",
    "    depths[node.orth_] = depth\n",
    "    if node.n_lefts + node.n_rights > 0:\n",
    "        return [walk_tree(child, depth+1) for child in node.children]\n",
    "[walk_tree(sent.root, 0) for sent in doc.sents]\n",
    "print(depths)\n",
    "print(max(depths.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1187966e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Al's principled leadership leadership pobj With\n",
      "the Department Department nsubj continue\n",
      "Justice Justice pobj of\n",
      "this important mission mission dobj continue\n",
      "the security security dobj defend\n",
      "all Americans Americans pobj of\n",
      "the liberty liberty conj Americans\n",
      "all Americans Americans pobj of\n"
     ]
    }
   ],
   "source": [
    "# This prints noun phrases\n",
    "for chunk in doc.noun_chunks:\n",
    "    print(chunk.text, chunk.root.text, chunk.root.dep_,\n",
    "            chunk.root.head.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d8ed9663",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "With prep continue VERB [leadership]\n",
      "Al poss leadership NOUN ['s]\n",
      "'s case Al PROPN []\n",
      "principled amod leadership NOUN []\n",
      "leadership pobj With ADP [Al, principled]\n",
      ", punct continue VERB []\n",
      "the det Department PROPN []\n",
      "Department nsubj continue VERB [the, of]\n",
      "of prep Department PROPN [Justice]\n",
      "Justice pobj of ADP []\n",
      "will aux continue VERB []\n",
      "continue ROOT continue VERB [With, ,, Department, will, mission, and, defend, .]\n",
      "this det mission NOUN []\n",
      "important amod mission NOUN []\n",
      "mission dobj continue VERB [this, important]\n",
      "and cc continue VERB []\n",
      "will aux defend VERB []\n",
      "defend conj continue VERB [will, security]\n",
      "the det security NOUN []\n",
      "security dobj defend VERB [the, of]\n",
      "of prep security NOUN [Americans]\n",
      "all det Americans PROPN []\n",
      "Americans pobj of ADP [all, and, liberty]\n",
      "and cc Americans PROPN []\n",
      "the det liberty NOUN []\n",
      "liberty conj Americans PROPN [the, of]\n",
      "of prep liberty NOUN [Americans]\n",
      "all det Americans PROPN []\n",
      "Americans pobj of ADP [all]\n",
      ". punct continue VERB []\n"
     ]
    }
   ],
   "source": [
    "# Navigating the tree - from Spacy's site\n",
    "for token in doc:\n",
    "    print(token.text, token.dep_, token.head.text, token.head.pos_,\n",
    "            [child for child in token.children])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f3180a6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8]\n"
     ]
    }
   ],
   "source": [
    "def walk_tree_depth(node, depth):\n",
    "    if node.n_lefts + node.n_rights > 0:\n",
    "        return max(walk_tree_depth(child, depth+1) for child in node.children )\n",
    "    else:\n",
    "        return depth\n",
    "    \n",
    "print([walk_tree_depth(sent.root, 0) for sent in doc.sents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ff27b340",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[walk_tree_depth(sent.root, 0) for sent in doc.sents][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a92b9cac",
   "metadata": {},
   "source": [
    "#### This makes the actual depth dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e5de79a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make data frame of sentences and parse tree depth of each\n",
    "def walk_tree_depth(node, depth):\n",
    "    if node.n_lefts + node.n_rights > 0:\n",
    "        return max(walk_tree_depth(child, depth+1) for child in node.children )\n",
    "    else:\n",
    "        return depth\n",
    "    \n",
    "tree_depth = pd.DataFrame(columns = ['date', 'source', 'sentence', 'depth'])\n",
    "for i, speech in enumerate(text_df['sentences']):\n",
    "    for j, sentence in enumerate(speech):\n",
    "        doc = nlp(sentence)\n",
    "        depth = [walk_tree_depth(sent.root, 0) for sent in doc.sents][0]\n",
    "        tree_depth.loc[len(tree_depth)] = [text_df['date'].iloc[i], text_df['source'].iloc[i], sentence, depth]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dee528b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_depth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46011c41",
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_depth.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7522b484",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tree_depth.to_csv('./Data/genData/sentence_depth_gwb.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab175b99",
   "metadata": {},
   "source": [
    "### Let's try spacy again for visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5aae67a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "doc = nlp(\"For everywhere in this country, there are first steps to be taken, there’s new ground to cover, there are more bridges to be crossed.\")\n",
    "# Since this is an interactive Jupyter environment, we can use displacy.render here\n",
    "displacy.render(doc, style='dep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7df9dc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "for token in doc:\n",
    "    print(token.text, token.dep_, token.head.text, token.head.pos_,\n",
    "            [child for child in token.children])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc25a169",
   "metadata": {},
   "outputs": [],
   "source": [
    "for chunk in doc.noun_chunks:\n",
    "    print(chunk.text, chunk.root.text, chunk.root.dep_,\n",
    "            chunk.root.head.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38b36c85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Finding a verb with a subject from below — good\n",
    "from spacy.symbols import nsubj, VERB\n",
    "verbs = set()\n",
    "for possible_subject in doc:\n",
    "    if possible_subject.dep == nsubj and possible_subject.head.pos == VERB:\n",
    "        verbs.add(possible_subject.head)\n",
    "print(verbs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69d2978a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print([token.text for token in doc[2].lefts])  # ['bright', 'red']\n",
    "print([token.text for token in doc[2].rights])  # ['on']\n",
    "print(doc[2].n_lefts)  # 2\n",
    "print(doc[2].n_rights)  # 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78fc45fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "root = [token for token in doc if token.head == token][0]\n",
    "subject = list(root.lefts)[0]\n",
    "for descendant in subject.subtree:\n",
    "    assert subject is descendant or subject.is_ancestor(descendant)\n",
    "    print(descendant.text, descendant.dep_, descendant.n_lefts,\n",
    "            descendant.n_rights,\n",
    "            [ancestor.text for ancestor in descendant.ancestors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4510a3b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "displacy.serve(doc, style=\"dep\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71d8b92f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}