pymupdf4llm_data_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-2zcuRnq4Wy5",
        "outputId": "2abba12a-6ce1-47a6-abd2-d8863ab8a4fb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pymupdf4llm\n",
            "  Downloading pymupdf4llm-0.0.17-py3-none-any.whl.metadata (4.1 kB)\n",
            "Collecting pymupdf>=1.24.10 (from pymupdf4llm)\n",
            "  Downloading PyMuPDF-1.24.13-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (3.4 kB)\n",
            "Downloading pymupdf4llm-0.0.17-py3-none-any.whl (26 kB)\n",
            "Downloading PyMuPDF-1.24.13-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (19.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.8/19.8 MB\u001b[0m \u001b[31m61.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pymupdf, pymupdf4llm\n",
            "Successfully installed pymupdf-1.24.13 pymupdf4llm-0.0.17\n"
          ]
        }
      ],
      "source": [
        "!pip install pymupdf4llm"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pymupdf4llm"
      ],
      "metadata": {
        "id": "uAC33zML6mpF"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "md_text = pymupdf4llm.to_markdown(\"test.pdf\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EGX0yS8R6pYJ",
        "outputId": "108b77dd-7e9c-4320-b1ea-882ff3a3ddef"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing test.pdf...\n",
            "[                                        ] (0/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b=============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[=============                           ] (1/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b=============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[==========================              ] (2/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b==============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (3/3)\b\b\b\b\b\b\b]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(md_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vXW72f9T60Nd",
        "outputId": "2785d6cc-69cb-4edc-9a53-41b893ae7d89"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# Gastroesophageal Surgery Case\n",
            "\n",
            "\n",
            "# A 76-Year-Old Woman  with Stomach Cancer\n",
            "\n",
            "JOHN MULLEN, MD\n",
            "_Massachusetts General Hospital_\n",
            "\n",
            "## PRESENTATION OF CASE\n",
            "\n",
            "A 76-year-old woman was seen in our\n",
            "multidisciplinary gastrointestinal oncology clinic\n",
            "for further management of an adenocarcinoma of\n",
            "the body of the stomach.\n",
            "\n",
            "The patient presented to her primary care\n",
            "physician with a four-week history of intermittent\n",
            "chest pressure radiating to her back. Extensive\n",
            "evaluation for a cardiac source was unremarkable,\n",
            "including a normal stress test and abdominal\n",
            "ultrasound. She then developed epigastric\n",
            "abdominal pain, and given that she carried\n",
            "a history of prior H. pylori infection in 2003,\n",
            "there was concern that she might have a peptic\n",
            "ulcer. An upper endoscopy was obtained and\n",
            "showed a large ulcerated tumor in the body of\n",
            "the stomach (Figure 1). Pathological examination\n",
            "of biopsy specimens of the stomach ulcer\n",
            "showed moderately to poorly differentiated\n",
            "\n",
            "**FIGURE 1. Image from upper endoscopic examination**\n",
            "demonstrating a large ulcerated tumor in the body\n",
            "of the stomach.\n",
            "\n",
            "\n",
            "adenocarcinoma. Staging computed tomography\n",
            "(CT) scans of the chest, abdomen and pelvis\n",
            "only showed prominent lymph nodes adjacent\n",
            "to the lesser curvature of the stomach within\n",
            "the gastrohepatic ligament and no evidence of\n",
            "metastatic disease.\n",
            "\n",
            "Upon evaluation at our cancer center, she reported\n",
            "intermittent upper abdominal discomfort that\n",
            "was worse without food but was typically relieved\n",
            "with eating. She denied any change in appetite or\n",
            "weight. She had no nausea, vomiting, early satiety\n",
            "or change in her bowel habits.\n",
            "\n",
            "Her past medical history was notable for type\n",
            "II diabetes mellitus, hypertension, elevated\n",
            "lipids, gastroesophageal reflux disease (GERD),\n",
            "diverticulosis, endometriosis, kidney stones,\n",
            "anxiety and osteopenia. Her medications included\n",
            "atorvastatin, Inderal, lisinopril, metformin,\n",
            "omeprazole and Paxil. She had allergies to dyazide,\n",
            "morphine, intravenous contrast dye, aspirin and\n",
            "triamcinolone cream. She was a retired research\n",
            "assistant and was married with two children. She\n",
            "never smoked and did not drink alcohol. There\n",
            "was no family history of stomach cancer, but her\n",
            "older brother died of colon cancer metastatic to\n",
            "the liver and lung at the age of 90.\n",
            "\n",
            "On examination, the weight was 182 pounds, the\n",
            "blood pressure 182/90 mm Hg, the pulse 90 beats\n",
            "per minute and the temperature 97.2 degrees\n",
            "Fahrenheit. There was mild epigastric tenderness\n",
            "without a palpable mass nor an enlarged liver.\n",
            "Results of a complete blood count, plasma levels\n",
            "\n",
            "\n",
            "-----\n",
            "\n",
            "of electrolytes and tests of kidney and liver\n",
            "function were normal.\n",
            "\n",
            "## DISCUSSION OF MANAGEMENT\n",
            "\n",
            "This patient has a stomach cancer that appears\n",
            "to be locoregionally advanced on imaging studies.\n",
            "The first decision regarding management is\n",
            "whether to resect it immediately or to administer\n",
            "preoperative chemotherapy, with or without\n",
            "preoperative radiation therapy.\n",
            "\n",
            "**Treatment of Locoregionally Advanced**\n",
            "**Gastric Cancer**\n",
            "\n",
            "Although surgery is the mainstay of curative\n",
            "treatment for gastric cancer, even after complete\n",
            "resection, more than half of patients with\n",
            "locally advanced tumors recur, and fewer than\n",
            "40% survive beyond three years[1]. Accordingly,\n",
            "investigators around the world have explored a\n",
            "variety of adjuvant and neoadjuvant multimodality\n",
            "treatment approaches to this disease. In North\n",
            "America, the standard approach has long\n",
            "been upfront surgery followed by adjuvant\n",
            "chemoradiotherapy, based on the findings of\n",
            "the Intergroup 0116 trial, which showed that\n",
            "postoperative 5-fluorouracil (5-FU)/leucovorinbased chemoradiation increases overall survival in\n",
            "comparison to surgery alone.[2] However, another\n",
            "standard of care for resectable gastric cancer in\n",
            "North America and Europe emerged with the\n",
            "publication of the MAGIC trial[3] a few years later\n",
            "and the FNCLCC/FFCD multicenter phase III trial[4]\n",
            "a few years after that. The MAGIC trial reported an\n",
            "improvement in overall survival with perioperative\n",
            "epirubicin, cisplatin and 5-FU (ECF) compared\n",
            "to surgery alone, and the FNCLCC/FFCD trial\n",
            "showed an improved R0 resection rate and survival\n",
            "with perioperative 5-FU and cisplatin.\n",
            "\n",
            "\n",
            "## To refer a patient or for a consultation,  visit our physician resources site at  massgeneral.org/GES or call 617-724-1020.\n",
            "\n",
            "Given this patient’s advanced age and\n",
            "medical co-morbidities, it was decided by the\n",
            "multidisciplinary team of surgical oncology,\n",
            "medical oncology and radiation oncology that the\n",
            "safest approach would be upfront surgery followed\n",
            "by postoperative chemotherapy and radiation\n",
            "therapy. The patient was felt to be at high risk for\n",
            "complications from either ECF chemotherapy\n",
            "or even a milder regimen of 5-FU, leucovorin\n",
            "and oxaliplatin (FOLFOX) chemotherapy, and\n",
            "we did not want to preclude the most important\n",
            "component of her treatment for potential cure –\n",
            "that being a gastrectomy.\n",
            "\n",
            "Accordingly, the patient was cleared for surgery\n",
            "by her cardiologist and primary care physician\n",
            "and was taken to the operating room for a\n",
            "subtotal gastrectomy and D2 lymphadenectomy.\n",
            "Since gastric cancer frequently spreads to the\n",
            "regional lymph nodes, and since at least 16 lymph\n",
            "nodes should be removed and examined by the\n",
            "pathologist for the most accurate staging of gastric\n",
            "cancer[1], we routinely remove not only the lymph\n",
            "nodes immediately adjacent to the stomach (the\n",
            "so-called D1 lymph nodes, in stations 1-6), but\n",
            "we also remove the second tier of lymph nodes\n",
            "along the major blood vessels to the liver, stomach\n",
            "and spleen (the so-called D2 lymph nodes, in\n",
            "\n",
            "**FIGURE 2. Lymph node stations including (A) the**\n",
            "perigastric, or D1, lymph nodes (stations 1 to 6), and (B)\n",
            "the regional, or D2 and D3, lymph nodes (stations 7-16).\n",
            "\n",
            "\n",
            "-----\n",
            "\n",
            "stations 7-12) (Figures 2A and 2B)[6]. In fact, since\n",
            "2006, we have been performing this technically\n",
            "demanding “D2 lymphadenectomy” with a low\n",
            "risk of complications and no deaths, and the\n",
            "median number of examined lymph nodes was\n",
            "40, and more than 90% of patients had at least 16\n",
            "lymph nodes examined[5]. In this patient’s particular\n",
            "operation, we removed approximately 80% of the\n",
            "stomach and reconstructed the gastrointestinal\n",
            "tract with a Roux-en-Y gastrojejunostomy\n",
            "(Figure 3)[6]. This procedure was accomplished\n",
            "through an open approach via an upper midline\n",
            "abdominal incision and was uncomplicated. Her\n",
            "postoperative course was unremarkable, and she\n",
            "was discharged to home on postoperative day\n",
            "number six, eating a soft diet composed of six\n",
            "small meals spaced evenly throughout the day.\n",
            "\n",
            "**FIGURE 3. Reconstruction by a retrocolic,**\n",
            "Roux-en-Y gastrojejunostomy.\n",
            "\n",
            "**Pathological Discussion and Role of Adjuvant**\n",
            "**Chemoradiation Therapy**\n",
            "\n",
            "Pathological examination showed a 4 cm tumor\n",
            "in the body of the stomach that was moderately\n",
            "to poorly differentiated and was staged as pT3\n",
            "\n",
            "\n",
            "## To refer a patient or for a consultation,  visit our physician resources site at  massgeneral.org/GES or call 617-724-1020.\n",
            "\n",
            "(tumor penetrates subserosa) and pN2 (3 nodes\n",
            "positive out of 31 nodes examined), or stage IIIA[1].\n",
            "All margins were negative, and there was evidence\n",
            "of both perineural and lymphovascular invasion.\n",
            "\n",
            "Based on the advanced nature of her cancer\n",
            "and the results of the Intergroup 0116 trial,\n",
            "which showed a survival advantage for adjuvant\n",
            "chemoradiation therapy, we treated her with\n",
            "eight cycles of FOLFOX chemotherapy followed\n",
            "by five weeks of 5-FU-based chemoradiation\n",
            "therapy to 45 Gray. She tolerated all of this therapy\n",
            "remarkably well and remains free of disease\n",
            "today. Her chance of long-term survival is on the\n",
            "order of 30%, and it is in large part because of the\n",
            "coordinated efforts of our team that specializes\n",
            "in gastric cancer that this patient made it through\n",
            "all stages of her treatment and thus has the best\n",
            "possible chance for a cure.\n",
            "\n",
            "## REFERENCES\n",
            "\n",
            "_1. Edge SB, Byrd DR, Compton CC, et al. AJCC cancer staging_\n",
            "_manual. 7th ed. New York: Springer-Verlag;2009:117-126._\n",
            "\n",
            "_2. Macdonald JS, Smalley SR, Benedetti J, et al._\n",
            "_Chemoradiotherapy after surgery compared with surgery alone_\n",
            "_for adenocarcinoma of the stomach or gastroesophageal_\n",
            "_junction. N Engl J Med. 2001;345:725-730._\n",
            "\n",
            "_3. Cunningham D, Allum WH, Stenning SP, et al. Perioperative_\n",
            "_chemotherapy versus surgery alone for resectable_\n",
            "_gastroesophageal cancer. N Engl J Med. 2006;355:11-20._\n",
            "\n",
            "_4. Ychou M, Boige V, Pignon JP, et al. Perioperative chemotherapy_\n",
            "_compared with surgery alone for resectable gastroesophageal_\n",
            "_adenocarcinoma: an FNCLCC and FFCD multicenter phase III_\n",
            "_trial. J Clin Oncol. 2011;29:1715-1721._\n",
            "\n",
            "_5. Schmidt B, Chang KK, Maduekwe UN, et al. D2_\n",
            "_lymphadenectomy with surgical ex vivo dissection into node_\n",
            "_stations for gastric adenocarcinoma can be performed safely in_\n",
            "_Western patients and ensures optimal staging. Ann Surg Oncol_\n",
            "_2013;20:2991-2999._\n",
            "\n",
            "_6. Mullen JT. Subtotal gastrectomy for gastric cancer. In: Fischer_\n",
            "_JE, Bland KI, Callery MP, Clagett GP, Jones DB, editors. Mastery of_\n",
            "_Surgery. Philadelphia: Lippincott, Williams, and Wilkins; Sixth_\n",
            "_edition, 2012._\n",
            "\n",
            "\n",
            "-----\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save Markdown to a File"
      ],
      "metadata": {
        "id": "VYWaY8ls7H6m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pathlib"
      ],
      "metadata": {
        "id": "LPTQFDvA63BZ"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pathlib.Path(\"output.md\").write_bytes(md_text.encode())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2p2iqgyl68lx",
        "outputId": "822b025f-186b-4e51-b5e9-7ae3a7b71d5a"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9013"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extract only specific pages"
      ],
      "metadata": {
        "id": "imCmEUdG7Cft"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "md_text_pages = pymupdf4llm.to_markdown(\"test.pdf\", pages=[1, 2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T4SRrN0H6_aJ",
        "outputId": "3abfb35a-c7c3-476d-c1c7-13e5e61235ed"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing test.pdf...\n",
            "[                                        ] (0/2)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b====================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[====================                    ] (1/2)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b====================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (2/2)\b\b\b\b\b\b\b]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "md_text_pages"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "id": "XYh_Woa-7MEK",
        "outputId": "e24cbe71-efe1-42ba-cb51-77b215058618"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'of electrolytes and tests of kidney and liver\\nfunction were normal.\\n\\n## DISCUSSION OF MANAGEMENT\\n\\nThis patient has a stomach cancer that appears\\nto be locoregionally advanced on imaging studies.\\nThe first decision regarding management is\\nwhether to resect it immediately or to administer\\npreoperative chemotherapy, with or without\\npreoperative radiation therapy.\\n\\n**Treatment of Locoregionally Advanced**\\n**Gastric Cancer**\\n\\nAlthough surgery is the mainstay of curative\\ntreatment for gastric cancer, even after complete\\nresection, more than half of patients with\\nlocally advanced tumors recur, and fewer than\\n40% survive beyond three years[1]. Accordingly,\\ninvestigators around the world have explored a\\nvariety of adjuvant and neoadjuvant multimodality\\ntreatment approaches to this disease. In North\\nAmerica, the standard approach has long\\nbeen upfront surgery followed by adjuvant\\nchemoradiotherapy, based on the findings of\\nthe Intergroup 0116 trial, which showed that\\npostoperative 5-fluorouracil (5-FU)/leucovorinbased chemoradiation increases overall survival in\\ncomparison to surgery alone.[2] However, another\\nstandard of care for resectable gastric cancer in\\nNorth America and Europe emerged with the\\npublication of the MAGIC trial[3] a few years later\\nand the FNCLCC/FFCD multicenter phase III trial[4]\\na few years after that. The MAGIC trial reported an\\nimprovement in overall survival with perioperative\\nepirubicin, cisplatin and 5-FU (ECF) compared\\nto surgery alone, and the FNCLCC/FFCD trial\\nshowed an improved R0 resection rate and survival\\nwith perioperative 5-FU and cisplatin.\\n\\n\\n## To refer a patient or for a consultation,  visit our physician resources site at  massgeneral.org/GES or call 617-724-1020.\\n\\nGiven this patient’s advanced age and\\nmedical co-morbidities, it was decided by the\\nmultidisciplinary team of surgical oncology,\\nmedical oncology and radiation oncology that the\\nsafest approach would be upfront surgery followed\\nby postoperative chemotherapy and radiation\\ntherapy. The patient was felt to be at high risk for\\ncomplications from either ECF chemotherapy\\nor even a milder regimen of 5-FU, leucovorin\\nand oxaliplatin (FOLFOX) chemotherapy, and\\nwe did not want to preclude the most important\\ncomponent of her treatment for potential cure –\\nthat being a gastrectomy.\\n\\nAccordingly, the patient was cleared for surgery\\nby her cardiologist and primary care physician\\nand was taken to the operating room for a\\nsubtotal gastrectomy and D2 lymphadenectomy.\\nSince gastric cancer frequently spreads to the\\nregional lymph nodes, and since at least 16 lymph\\nnodes should be removed and examined by the\\npathologist for the most accurate staging of gastric\\ncancer[1], we routinely remove not only the lymph\\nnodes immediately adjacent to the stomach (the\\nso-called D1 lymph nodes, in stations 1-6), but\\nwe also remove the second tier of lymph nodes\\nalong the major blood vessels to the liver, stomach\\nand spleen (the so-called D2 lymph nodes, in\\n\\n**FIGURE 2. Lymph node stations including (A) the**\\nperigastric, or D1, lymph nodes (stations 1 to 6), and (B)\\nthe regional, or D2 and D3, lymph nodes (stations 7-16).\\n\\n\\n-----\\n\\nstations 7-12) (Figures 2A and 2B)[6]. In fact, since\\n2006, we have been performing this technically\\ndemanding “D2 lymphadenectomy” with a low\\nrisk of complications and no deaths, and the\\nmedian number of examined lymph nodes was\\n40, and more than 90% of patients had at least 16\\nlymph nodes examined[5]. In this patient’s particular\\noperation, we removed approximately 80% of the\\nstomach and reconstructed the gastrointestinal\\ntract with a Roux-en-Y gastrojejunostomy\\n(Figure 3)[6]. This procedure was accomplished\\nthrough an open approach via an upper midline\\nabdominal incision and was uncomplicated. Her\\npostoperative course was unremarkable, and she\\nwas discharged to home on postoperative day\\nnumber six, eating a soft diet composed of six\\nsmall meals spaced evenly throughout the day.\\n\\n**FIGURE 3. Reconstruction by a retrocolic,**\\nRoux-en-Y gastrojejunostomy.\\n\\n**Pathological Discussion and Role of Adjuvant**\\n**Chemoradiation Therapy**\\n\\nPathological examination showed a 4 cm tumor\\nin the body of the stomach that was moderately\\nto poorly differentiated and was staged as pT3\\n\\n\\n## To refer a patient or for a consultation,  visit our physician resources site at  massgeneral.org/GES or call 617-724-1020.\\n\\n(tumor penetrates subserosa) and pN2 (3 nodes\\npositive out of 31 nodes examined), or stage IIIA[1].\\nAll margins were negative, and there was evidence\\nof both perineural and lymphovascular invasion.\\n\\nBased on the advanced nature of her cancer\\nand the results of the Intergroup 0116 trial,\\nwhich showed a survival advantage for adjuvant\\nchemoradiation therapy, we treated her with\\neight cycles of FOLFOX chemotherapy followed\\nby five weeks of 5-FU-based chemoradiation\\ntherapy to 45 Gray. She tolerated all of this therapy\\nremarkably well and remains free of disease\\ntoday. Her chance of long-term survival is on the\\norder of 30%, and it is in large part because of the\\ncoordinated efforts of our team that specializes\\nin gastric cancer that this patient made it through\\nall stages of her treatment and thus has the best\\npossible chance for a cure.\\n\\n## REFERENCES\\n\\n_1. Edge SB, Byrd DR, Compton CC, et al. AJCC cancer staging_\\n_manual. 7th ed. New York: Springer-Verlag;2009:117-126._\\n\\n_2. Macdonald JS, Smalley SR, Benedetti J, et al._\\n_Chemoradiotherapy after surgery compared with surgery alone_\\n_for adenocarcinoma of the stomach or gastroesophageal_\\n_junction. N Engl J Med. 2001;345:725-730._\\n\\n_3. Cunningham D, Allum WH, Stenning SP, et al. Perioperative_\\n_chemotherapy versus surgery alone for resectable_\\n_gastroesophageal cancer. N Engl J Med. 2006;355:11-20._\\n\\n_4. Ychou M, Boige V, Pignon JP, et al. Perioperative chemotherapy_\\n_compared with surgery alone for resectable gastroesophageal_\\n_adenocarcinoma: an FNCLCC and FFCD multicenter phase III_\\n_trial. J Clin Oncol. 2011;29:1715-1721._\\n\\n_5. Schmidt B, Chang KK, Maduekwe UN, et al. D2_\\n_lymphadenectomy with surgical ex vivo dissection into node_\\n_stations for gastric adenocarcinoma can be performed safely in_\\n_Western patients and ensures optimal staging. Ann Surg Oncol_\\n_2013;20:2991-2999._\\n\\n_6. Mullen JT. Subtotal gastrectomy for gastric cancer. In: Fischer_\\n_JE, Bland KI, Callery MP, Clagett GP, Jones DB, editors. Mastery of_\\n_Surgery. Philadelphia: Lippincott, Williams, and Wilkins; Sixth_\\n_edition, 2012._\\n\\n\\n-----\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extracting documents for LlamaIndex"
      ],
      "metadata": {
        "id": "RAgcyzUR7UBG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install llama_index"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WgbcD0ds7RPy",
        "outputId": "17501ea8-6f59-415f-fccd-13ce2619e7f1"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting llama_index\n",
            "  Downloading llama_index-0.11.23-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting llama-index-agent-openai<0.4.0,>=0.3.4 (from llama_index)\n",
            "  Downloading llama_index_agent_openai-0.3.4-py3-none-any.whl.metadata (728 bytes)\n",
            "Collecting llama-index-cli<0.4.0,>=0.3.1 (from llama_index)\n",
            "  Downloading llama_index_cli-0.3.1-py3-none-any.whl.metadata (1.5 kB)\n",
            "Collecting llama-index-core<0.12.0,>=0.11.23 (from llama_index)\n",
            "  Downloading llama_index_core-0.11.23-py3-none-any.whl.metadata (2.5 kB)\n",
            "Collecting llama-index-embeddings-openai<0.3.0,>=0.2.4 (from llama_index)\n",
            "  Downloading llama_index_embeddings_openai-0.2.5-py3-none-any.whl.metadata (686 bytes)\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.3.0 (from llama_index)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.4.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "Collecting llama-index-legacy<0.10.0,>=0.9.48 (from llama_index)\n",
            "  Downloading llama_index_legacy-0.9.48.post4-py3-none-any.whl.metadata (8.5 kB)\n",
            "Collecting llama-index-llms-openai<0.3.0,>=0.2.10 (from llama_index)\n",
            "  Downloading llama_index_llms_openai-0.2.16-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting llama-index-multi-modal-llms-openai<0.3.0,>=0.2.0 (from llama_index)\n",
            "  Downloading llama_index_multi_modal_llms_openai-0.2.3-py3-none-any.whl.metadata (729 bytes)\n",
            "Collecting llama-index-program-openai<0.3.0,>=0.2.0 (from llama_index)\n",
            "  Downloading llama_index_program_openai-0.2.0-py3-none-any.whl.metadata (766 bytes)\n",
            "Collecting llama-index-question-gen-openai<0.3.0,>=0.2.0 (from llama_index)\n",
            "  Downloading llama_index_question_gen_openai-0.2.0-py3-none-any.whl.metadata (785 bytes)\n",
            "Collecting llama-index-readers-file<0.4.0,>=0.3.0 (from llama_index)\n",
            "  Downloading llama_index_readers_file-0.3.0-py3-none-any.whl.metadata (5.4 kB)\n",
            "Collecting llama-index-readers-llama-parse>=0.3.0 (from llama_index)\n",
            "  Downloading llama_index_readers_llama_parse-0.3.0-py3-none-any.whl.metadata (3.5 kB)\n",
            "Collecting nltk>3.8.1 (from llama_index)\n",
            "  Downloading nltk-3.9.1-py3-none-any.whl.metadata (2.9 kB)\n",
            "Requirement already satisfied: openai>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-agent-openai<0.4.0,>=0.3.4->llama_index) (1.52.2)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (6.0.2)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.23->llama_index) (2.0.36)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (3.10.10)\n",
            "Collecting dataclasses-json (from llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (1.2.14)\n",
            "Collecting dirtyjson<2.0.0,>=1.0.8 (from llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading dirtyjson-1.0.8-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting filetype<2.0.0,>=1.2.0 (from llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading filetype-1.2.0-py2.py3-none-any.whl.metadata (6.5 kB)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (2024.10.0)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (0.27.2)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (3.4.2)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (1.26.4)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (10.4.0)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (2.9.2)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (2.32.3)\n",
            "Collecting tenacity!=8.4.0,<9.0.0,>=8.2.0 (from llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading tenacity-8.5.0-py3-none-any.whl.metadata (1.2 kB)\n",
            "Collecting tiktoken>=0.3.3 (from llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (4.66.6)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (4.12.2)\n",
            "Collecting typing-inspect>=0.8.0 (from llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.23->llama_index) (1.16.0)\n",
            "Collecting llama-cloud>=0.0.11 (from llama-index-indices-managed-llama-cloud>=0.3.0->llama_index)\n",
            "  Downloading llama_cloud-0.1.4-py3-none-any.whl.metadata (814 bytes)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-legacy<0.10.0,>=0.9.48->llama_index) (2.2.2)\n",
            "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file<0.4.0,>=0.3.0->llama_index) (4.12.3)\n",
            "Collecting pypdf<6.0.0,>=5.1.0 (from llama-index-readers-file<0.4.0,>=0.3.0->llama_index)\n",
            "  Downloading pypdf-5.1.0-py3-none-any.whl.metadata (7.2 kB)\n",
            "Collecting striprtf<0.0.27,>=0.0.26 (from llama-index-readers-file<0.4.0,>=0.3.0->llama_index)\n",
            "  Downloading striprtf-0.0.26-py3-none-any.whl.metadata (2.1 kB)\n",
            "Collecting llama-parse>=0.5.0 (from llama-index-readers-llama-parse>=0.3.0->llama_index)\n",
            "  Downloading llama_parse-0.5.13-py3-none-any.whl.metadata (6.9 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama_index) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama_index) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama_index) (2024.9.11)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (1.17.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (4.0.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file<0.4.0,>=0.3.0->llama_index) (2.6)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (2024.8.30)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (1.0.6)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (3.10)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (0.14.0)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.14.0->llama-index-agent-openai<0.4.0,>=0.3.4->llama_index) (1.9.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.14.0->llama-index-agent-openai<0.4.0,>=0.3.4->llama_index) (0.7.0)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.23->llama_index) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.23->llama_index) (2.23.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.23->llama_index) (3.4.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.23->llama_index) (2.2.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.23->llama_index) (3.1.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->llama-index-core<0.12.0,>=0.11.23->llama_index)\n",
            "  Downloading marshmallow-3.23.1-py3-none-any.whl.metadata (7.5 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-legacy<0.10.0,>=0.9.48->llama_index) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-legacy<0.10.0,>=0.9.48->llama_index) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-legacy<0.10.0,>=0.9.48->llama_index) (2024.2)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.23->llama_index) (1.2.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.12.0,>=0.11.23->llama_index) (24.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-index-legacy<0.10.0,>=0.9.48->llama_index) (1.16.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.23->llama_index) (0.2.0)\n",
            "Downloading llama_index-0.11.23-py3-none-any.whl (6.8 kB)\n",
            "Downloading llama_index_agent_openai-0.3.4-py3-none-any.whl (13 kB)\n",
            "Downloading llama_index_cli-0.3.1-py3-none-any.whl (27 kB)\n",
            "Downloading llama_index_core-0.11.23-py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m36.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_embeddings_openai-0.2.5-py3-none-any.whl (6.1 kB)\n",
            "Downloading llama_index_indices_managed_llama_cloud-0.4.0-py3-none-any.whl (10 kB)\n",
            "Downloading llama_index_legacy-0.9.48.post4-py3-none-any.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m65.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_llms_openai-0.2.16-py3-none-any.whl (13 kB)\n",
            "Downloading llama_index_multi_modal_llms_openai-0.2.3-py3-none-any.whl (5.9 kB)\n",
            "Downloading llama_index_program_openai-0.2.0-py3-none-any.whl (5.3 kB)\n",
            "Downloading llama_index_question_gen_openai-0.2.0-py3-none-any.whl (2.9 kB)\n",
            "Downloading llama_index_readers_file-0.3.0-py3-none-any.whl (38 kB)\n",
            "Downloading llama_index_readers_llama_parse-0.3.0-py3-none-any.whl (2.5 kB)\n",
            "Downloading nltk-3.9.1-py3-none-any.whl (1.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m73.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dirtyjson-1.0.8-py3-none-any.whl (25 kB)\n",
            "Downloading filetype-1.2.0-py2.py3-none-any.whl (19 kB)\n",
            "Downloading llama_cloud-0.1.4-py3-none-any.whl (176 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.8/176.8 kB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_parse-0.5.13-py3-none-any.whl (13 kB)\n",
            "Downloading pypdf-5.1.0-py3-none-any.whl (297 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m27.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading striprtf-0.0.26-py3-none-any.whl (6.9 kB)\n",
            "Downloading tenacity-8.5.0-py3-none-any.whl (28 kB)\n",
            "Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m67.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Downloading dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n",
            "Downloading marshmallow-3.23.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.5/49.5 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: striprtf, filetype, dirtyjson, tenacity, pypdf, nltk, mypy-extensions, marshmallow, typing-inspect, tiktoken, llama-cloud, dataclasses-json, llama-index-legacy, llama-index-core, llama-parse, llama-index-readers-file, llama-index-llms-openai, llama-index-indices-managed-llama-cloud, llama-index-embeddings-openai, llama-index-readers-llama-parse, llama-index-multi-modal-llms-openai, llama-index-cli, llama-index-agent-openai, llama-index-program-openai, llama-index-question-gen-openai, llama_index\n",
            "  Attempting uninstall: tenacity\n",
            "    Found existing installation: tenacity 9.0.0\n",
            "    Uninstalling tenacity-9.0.0:\n",
            "      Successfully uninstalled tenacity-9.0.0\n",
            "  Attempting uninstall: nltk\n",
            "    Found existing installation: nltk 3.8.1\n",
            "    Uninstalling nltk-3.8.1:\n",
            "      Successfully uninstalled nltk-3.8.1\n",
            "Successfully installed dataclasses-json-0.6.7 dirtyjson-1.0.8 filetype-1.2.0 llama-cloud-0.1.4 llama-index-agent-openai-0.3.4 llama-index-cli-0.3.1 llama-index-core-0.11.23 llama-index-embeddings-openai-0.2.5 llama-index-indices-managed-llama-cloud-0.4.0 llama-index-legacy-0.9.48.post4 llama-index-llms-openai-0.2.16 llama-index-multi-modal-llms-openai-0.2.3 llama-index-program-openai-0.2.0 llama-index-question-gen-openai-0.2.0 llama-index-readers-file-0.3.0 llama-index-readers-llama-parse-0.3.0 llama-parse-0.5.13 llama_index-0.11.23 marshmallow-3.23.1 mypy-extensions-1.0.0 nltk-3.9.1 pypdf-5.1.0 striprtf-0.0.26 tenacity-8.5.0 tiktoken-0.8.0 typing-inspect-0.9.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "llama_reader = pymupdf4llm.LlamaMarkdownReader()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVw27-KV7W45",
        "outputId": "8e225657-0651-4559-8a19-9be4eb30312c"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Successfully imported LlamaIndex\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "llama_docs = llama_reader.load_data(\"test.pdf\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IgCWftC-7dZ5",
        "outputId": "ac6220c6-a3f0-4e20-9ee3-03c1998879aa"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing test.pdf...\n",
            "[                                        ] (0/1)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b========================================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (1/1)\b\b\b\b\b\b\b]\n",
            "Processing test.pdf...\n",
            "[                                        ] (0/1)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b========================================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (1/1)\b\b\b\b\b\b\b]\n",
            "Processing test.pdf...\n",
            "[                                        ] (0/1)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b========================================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (1/1)\b\b\b\b\b\b\b]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"LlamaIndex Compatiable Data: {len(llama_docs)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "id-xQ4v-7jnA",
        "outputId": "26affc83-b924-4058-e6a9-29bb0ac337f6"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LlamaIndex Compatiable Data: 3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "{llama_docs[0].text[:500]}"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ndy2H4JP7mjY",
        "outputId": "088bb1f0-f1f3-4fa5-b907-cf916f602f73"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'# Gastroesophageal Surgery Case\\n\\n\\n# A 76-Year-Old Woman  with Stomach Cancer\\n\\nJOHN MULLEN, MD\\n_Massachusetts General Hospital_\\n\\n## PRESENTATION OF CASE\\n\\nA 76-year-old woman was seen in our\\nmultidisciplinary gastrointestinal oncology clinic\\nfor further management of an adenocarcinoma of\\nthe body of the stomach.\\n\\nThe patient presented to her primary care\\nphysician with a four-week history of intermittent\\nchest pressure radiating to her back. Extensive\\nevaluation for a cardiac source was unremarkab'}"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extracting images"
      ],
      "metadata": {
        "id": "n31LZzwW7roa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "md_text_images = pymupdf4llm.to_markdown(\n",
        "    doc=\"test.pdf\",\n",
        "    pages=[0, 2],\n",
        "    page_chunks=True,\n",
        "    write_images=True,\n",
        "    image_path=\"images\",\n",
        "    image_format=\"png\",\n",
        "    dpi=300\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y5MUO59n7pFY",
        "outputId": "0f4afe84-a3cd-4113-8260-e4f210710c62"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing test.pdf...\n",
            "[                                        ] (0/2)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b====================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[====================                    ] (1/2)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b====================\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (2/2)\b\b\b\b\b\b\b]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chunking data and extracting it with metadata"
      ],
      "metadata": {
        "id": "3zcCr4gL7zjx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "md_text_chunks = pymupdf4llm.to_markdown(\n",
        "    doc=\"test.pdf\",\n",
        "    pages=[0, 1, 2],\n",
        "    page_chunks=True\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Ua-dHxY7up5",
        "outputId": "26500f34-e35b-45bf-a7a8-c2e70b4d9eeb"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing test.pdf...\n",
            "[                                        ] (0/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b=============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[=============                           ] (1/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b=============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[==========================              ] (2/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b==============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (3/3)\b\b\b\b\b\b\b]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(md_text_chunks[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j_3eJ82x74IB",
        "outputId": "3e5d7aec-ae53-4aa3-e579-9a16eefa0e35"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'metadata': {'format': 'PDF 1.6', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'creator': 'Adobe InDesign CC (Macintosh)', 'producer': 'Adobe PDF Library 11.0', 'creationDate': \"D:20140225134206-06'00'\", 'modDate': \"D:20140225163520-05'00'\", 'trapped': '', 'encryption': None, 'file_path': 'test.pdf', 'page_count': 3, 'page': 1}, 'toc_items': [], 'tables': [], 'images': [{'number': 10, 'bbox': Rect(72.73251342773438, 548.9599609375, 263.70458984375, 694.3999633789062), 'transform': (190.97206115722656, 0.0, -0.0, 145.4399871826172, 72.73251342773438, 548.9599609375), 'width': 796, 'height': 606, 'colorspace': 4, 'cs-name': 'DeviceCMYK', 'xres': 96, 'yres': 96, 'bpc': 8, 'size': 129628}], 'graphics': [], 'text': '# Gastroesophageal Surgery Case\\n\\n\\n# A 76-Year-Old Woman  with Stomach Cancer\\n\\nJOHN MULLEN, MD\\n_Massachusetts General Hospital_\\n\\n## PRESENTATION OF CASE\\n\\nA 76-year-old woman was seen in our\\nmultidisciplinary gastrointestinal oncology clinic\\nfor further management of an adenocarcinoma of\\nthe body of the stomach.\\n\\nThe patient presented to her primary care\\nphysician with a four-week history of intermittent\\nchest pressure radiating to her back. Extensive\\nevaluation for a cardiac source was unremarkable,\\nincluding a normal stress test and abdominal\\nultrasound. She then developed epigastric\\nabdominal pain, and given that she carried\\na history of prior H. pylori infection in 2003,\\nthere was concern that she might have a peptic\\nulcer. An upper endoscopy was obtained and\\nshowed a large ulcerated tumor in the body of\\nthe stomach (Figure 1). Pathological examination\\nof biopsy specimens of the stomach ulcer\\nshowed moderately to poorly differentiated\\n\\n**FIGURE 1. Image from upper endoscopic examination**\\ndemonstrating a large ulcerated tumor in the body\\nof the stomach.\\n\\n\\nadenocarcinoma. Staging computed tomography\\n(CT) scans of the chest, abdomen and pelvis\\nonly showed prominent lymph nodes adjacent\\nto the lesser curvature of the stomach within\\nthe gastrohepatic ligament and no evidence of\\nmetastatic disease.\\n\\nUpon evaluation at our cancer center, she reported\\nintermittent upper abdominal discomfort that\\nwas worse without food but was typically relieved\\nwith eating. She denied any change in appetite or\\nweight. She had no nausea, vomiting, early satiety\\nor change in her bowel habits.\\n\\nHer past medical history was notable for type\\nII diabetes mellitus, hypertension, elevated\\nlipids, gastroesophageal reflux disease (GERD),\\ndiverticulosis, endometriosis, kidney stones,\\nanxiety and osteopenia. Her medications included\\natorvastatin, Inderal, lisinopril, metformin,\\nomeprazole and Paxil. She had allergies to dyazide,\\nmorphine, intravenous contrast dye, aspirin and\\ntriamcinolone cream. She was a retired research\\nassistant and was married with two children. She\\nnever smoked and did not drink alcohol. There\\nwas no family history of stomach cancer, but her\\nolder brother died of colon cancer metastatic to\\nthe liver and lung at the age of 90.\\n\\nOn examination, the weight was 182 pounds, the\\nblood pressure 182/90 mm Hg, the pulse 90 beats\\nper minute and the temperature 97.2 degrees\\nFahrenheit. There was mild epigastric tenderness\\nwithout a palpable mass nor an enlarged liver.\\nResults of a complete blood count, plasma levels\\n\\n\\n-----\\n\\n', 'words': []}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Detailed word-by-word extraction"
      ],
      "metadata": {
        "id": "96xh3zUa7-6l"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "md_text_words = pymupdf4llm.to_markdown(\n",
        "    doc=\"test.pdf\",\n",
        "    pages=[0, 1, 2],\n",
        "    page_chunks=True,\n",
        "    write_images=True,\n",
        "    image_path=\"images\",\n",
        "    image_format=\"png\",\n",
        "    dpi=300,\n",
        "    extract_words=True\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Ym58pBA78KI",
        "outputId": "e6f6e4a6-bd1e-4c84-928c-df82c9d0ae34"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Processing test.pdf...\n",
            "[                                        ] (0/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b=============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[=============                           ] (1/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b=============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[==========================              ] (2/3)\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b==============\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b[========================================] (3/3)\b\b\b\b\b\b\b]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(md_text_words[0]['words'][:5])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Nmu_tes8B8B",
        "outputId": "1f19d6c3-1926-49c1-e14b-061478496b8f"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[(73.0, 116.09974670410156, 236.95803833007812, 136.09974670410156, 'Gastroesophageal', 9, 0, 0), (241.59803771972656, 116.09974670410156, 314.39605712890625, 136.09974670410156, 'Surgery', 9, 0, 1), (319.03607177734375, 116.09974670410156, 363.0960388183594, 136.09974670410156, 'Case', 9, 0, 2), (73.80000305175781, 161.99977111816406, 86.56000518798828, 181.99977111816406, 'A', 0, 0, 0), (91.20000457763672, 161.99977111816406, 201.1380157470703, 181.99977111816406, '76-Year-Old', 0, 0, 1)]\n"
          ]
        }
      ]
    }
  ]
}