Mitteilungsblatt_with_Flair_and_evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Mitteilungsblatt\n",
        "\n",
        "NER analysis"
      ],
      "metadata": {
        "id": "Ff7eedhabXYy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Mount google drive"
      ],
      "metadata": {
        "id": "DI1ZzS-thhiJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "YOPIyn6Yhcl4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load the OCRed text from the folder"
      ],
      "metadata": {
        "id": "npG3baxoblcX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C9RPbn3VZ-L2"
      },
      "outputs": [],
      "source": [
        "from genericpath import isdir\n",
        "import os\n",
        "\n",
        "path = \"/content/drive/MyDrive/OCR results/OCR4all results/1936-01-I/complete.txt\"\n",
        "\n",
        "if os.path.isdir(path):\n",
        "  for file in os.listdir(path):\n",
        "    with open(file, \"r\") as txt_file:\n",
        "      lines = txt_file.readlines()\n",
        "\n",
        "else:\n",
        "  file = path\n",
        "  with open(file, \"r\") as txt_file:\n",
        "      text = txt_file.read()\n",
        "    \n",
        "print(text)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run spacy on the text"
      ],
      "metadata": {
        "id": "RFcagh22nZ84"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "download spacy and the model from the network"
      ],
      "metadata": {
        "id": "1KKN8o4bnknh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade spacy\n",
        "!python -m spacy download de_core_news_lg"
      ],
      "metadata": {
        "id": "osSdDeB9ndnF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "import spacy"
      ],
      "metadata": {
        "id": "g1kVUENbpnvC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy \n",
        "nlp = spacy.load('de_core_news_lg') \n",
        "print(\"Done\")"
      ],
      "metadata": {
        "id": "EgqEw65Sn2cJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "doc = nlp(text)\n",
        "\n",
        "entity_list = []\n",
        "\n",
        "for ent in doc.ents:\n",
        "   #print(ent.text, ent.start_char, ent.end_char, ent.label_)\n",
        "   if ent.label_ == 'PER':\n",
        "     entity_list.append((ent.text, ent.label_))\n",
        "\n",
        "for entity in entity_list:\n",
        "  print(entity)\n"
      ],
      "metadata": {
        "id": "qqls6Pxzp0su"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from spacy import displacy\n",
        "\n",
        "displacy.render(doc, style='ent', jupyter=True, options={'distance': 90})\n"
      ],
      "metadata": {
        "id": "nUOBUU1Yp_O6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Adding evaluation pipeline"
      ],
      "metadata": {
        "id": "yNCTznfkb7I2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
        "import re"
      ],
      "metadata": {
        "id": "4EfzxtJJb91T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Getting person spans from spacy model"
      ],
      "metadata": {
        "id": "xQhezmqXcBd7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "path = \"/content/drive/MyDrive/OCR results/Samples4Markup/Sample1_for_Markup_issue_1936_01\"\n",
        "with open(path, 'r') as openfile:\n",
        "    text = openfile.read()"
      ],
      "metadata": {
        "id": "jLzN8Kr4bH4N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_spans(ner_model, some_text):\n",
        "    parsed_text = ner_model(some_text)\n",
        "    spans = []\n",
        "    for entity in parsed_text.ents:\n",
        "        entity_type = entity.label_\n",
        "        entity_string = entity.text\n",
        "        if entity_type == 'PER':\n",
        "            spans.append((entity.start_char, entity.end_char))\n",
        "    return spans"
      ],
      "metadata": {
        "id": "uLOoqojCbqHz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spacy_spans = get_spans(nlp, text)"
      ],
      "metadata": {
        "id": "UdZLoMAocHPt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spacy_spans[:10]"
      ],
      "metadata": {
        "id": "0EprDmm3cKfG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_binary_prediction(spans, textlen):\n",
        "    spacy_binary_prediction = pd.Series(np.zeros(textlen))\n",
        "    for span in spans:\n",
        "        spacy_binary_prediction.loc[span[0]:span[1]] = 1\n",
        "    return spacy_binary_prediction"
      ],
      "metadata": {
        "id": "FibEM1A-cPc9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spacy_binary_prediction = get_binary_prediction(spacy_spans, len(text))"
      ],
      "metadata": {
        "id": "fSegUuRLkGsF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spacy_binary_prediction [80:110]"
      ],
      "metadata": {
        "id": "VTH1kzcjgw1C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Getting person spans from manual markup"
      ],
      "metadata": {
        "id": "sFXvr0IfcTtL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "manual_path = \"/content/drive/MyDrive/OCR results/Samples4Markup/ManualMarkup/Sample1_for_Markup_issue_1936_01_Default_Annotations.xml\""
      ],
      "metadata": {
        "id": "-xl-UJxDcR05"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_annotation_spans_from_xml(annotation_file_path):\n",
        "    with open(annotation_file_path) as openfile:\n",
        "        soup = BeautifulSoup(openfile, 'xml')\n",
        "    annotations = soup.find_all('seg')\n",
        "    annotation_pointers = [seg.find('ptr') for seg in annotations]\n",
        "    annotation_borders = [get_borders(ptr) for ptr in annotation_pointers]\n",
        "    manual_target = pd.Series(np.zeros(get_file_len_from_xml(soup)))\n",
        "    for span in annotation_borders:\n",
        "        manual_target.loc[span[0]:span[1]] = 1\n",
        "    return manual_target"
      ],
      "metadata": {
        "id": "PiGepNpxcXlt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_borders(ptr):\n",
        "    target = ptr['target']\n",
        "    border_range = re.search(r'(.+)char=(.+)', target).group(2)\n",
        "    border_range = border_range.split(',')\n",
        "    start = int(border_range[0])\n",
        "    end = int(border_range[1])\n",
        "    return start, end"
      ],
      "metadata": {
        "id": "U9SHFPnLcYE4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_file_len_from_xml(soup):\n",
        "    filelen = 0\n",
        "    all_pointers = soup.find_all('ptr')\n",
        "    for pointer in all_pointers:\n",
        "        start, end = get_borders(pointer)\n",
        "        if end > filelen:\n",
        "            filelen = end\n",
        "    return filelen"
      ],
      "metadata": {
        "id": "QJtKFHWucZxH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "target = get_annotation_spans_from_xml(manual_path)"
      ],
      "metadata": {
        "id": "Jo-305excbLV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "target[90:110]"
      ],
      "metadata": {
        "id": "sBi4li2acc4q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "target.value_counts()"
      ],
      "metadata": {
        "id": "9q_GRDGbcfrp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Measuring"
      ],
      "metadata": {
        "id": "uDnWuH5cclUc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "precision_score(target, spacy_binary_prediction)"
      ],
      "metadata": {
        "id": "S7zoG493cjXa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(target, spacy_binary_prediction)"
      ],
      "metadata": {
        "id": "zxJ3ykKXcmHA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f1_score(target, spacy_binary_prediction)"
      ],
      "metadata": {
        "id": "KaATDHmJcqW3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Improving spacy with some ad hoc rules"
      ],
      "metadata": {
        "id": "1v7TIp7jc2eJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#some cludgy spaghetti ad hoc filter \n",
        "\n",
        "def is_fake_person(some_string):\n",
        "    lower_string = some_string.lower()\n",
        "    if lower_string.endswith('str'):\n",
        "        return True\n",
        "    elif lower_string.endswith('boulevard'): \n",
        "        return True\n",
        "    elif lower_string.endswith('ave.'): \n",
        "        return True\n",
        "    elif lower_string.endswith('bvd.'): \n",
        "        return True\n",
        "    elif lower_string.endswith('gen'): \n",
        "        return True\n",
        "    elif lower_string.endswith('ung'): \n",
        "        return True\n",
        "    #elif some_string.endswith('P. O. B'): \n",
        "    #    return True \n",
        "    elif 'P.O.B.' in some_string or 'P. O. B' in some_string:\n",
        "        return True\n",
        "    elif some_string.endswith('H. O. G.'): \n",
        "        return True \n",
        "    elif re.search('[למנסעפצקרשתםןףץאבגדהוזחטיכך]', lower_string) is not None:\n",
        "        return True\n",
        "    elif re.search('[A-Z]', some_string) is None:\n",
        "        return True\n",
        "    elif ' ' not in some_string:\n",
        "        return True\n",
        "    elif 'strasse' in lower_string:\n",
        "        return True\n",
        "    return False"
      ],
      "metadata": {
        "id": "1dXrA3jncsGV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_spans_with_filter(ner_model, some_text):\n",
        "    parsed_text = ner_model(some_text)\n",
        "    spans = []\n",
        "    for entity in parsed_text.ents:\n",
        "        entity_type = entity.label_\n",
        "        entity_string = entity.text\n",
        "        if entity_type == 'PER' and not is_fake_person(entity_string):\n",
        "            spans.append((entity.start_char, entity.end_char))\n",
        "    return spans"
      ],
      "metadata": {
        "id": "CNxx2ALXdZ8y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spacy_spans = get_spans_with_filter(nlp, text)"
      ],
      "metadata": {
        "id": "rsqVxeOEjXwz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spacy_binary_prediction = get_binary_prediction(spacy_spans, len(text))"
      ],
      "metadata": {
        "id": "Th0UQM8Rj5vy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Measuring the improvement"
      ],
      "metadata": {
        "id": "G4hH4Gy711ZS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "precision_score(target, spacy_binary_prediction)"
      ],
      "metadata": {
        "id": "NkzpXLZFk0CG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(target, spacy_binary_prediction)"
      ],
      "metadata": {
        "id": "Fg2zAxV_k2F1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f1_score(target, spacy_binary_prediction)"
      ],
      "metadata": {
        "id": "t8VpUMD5l_sY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Getting person spans from Flair model"
      ],
      "metadata": {
        "id": "SAydXACmmVYL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install flair"
      ],
      "metadata": {
        "id": "kzlx5RD0zS7J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from flair.data import Sentence\n",
        "from flair.models import SequenceTagger"
      ],
      "metadata": {
        "id": "nwuzMhGdmBOd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tagger = SequenceTagger.load(\"flair/ner-german-large\")"
      ],
      "metadata": {
        "id": "wvRkTagXzWA7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_spans_flair(flair_model, some_text):\n",
        "    # load tagger\n",
        "    predicted_spans = []\n",
        "\n",
        "    # make example sentence\n",
        "    sentence = Sentence(some_text)\n",
        "\n",
        "    # predict NER tags\n",
        "    flair_model.predict(sentence)\n",
        "\n",
        "    # print sentence\n",
        "    #print(sentence)\n",
        "\n",
        "    # print predicted NER spans\n",
        "    #print('The following NER tags are found:')\n",
        "    # iterate over entities and print\n",
        "    for entity in sentence.get_spans('ner'):\n",
        "        entity_type = entity.tag\n",
        "        entity_string = entity.text\n",
        "        if entity_type == 'PER':\n",
        "            #print(entity_string)\n",
        "            predicted_spans.append((entity.start_position, entity.end_position))\n",
        "    return predicted_spans"
      ],
      "metadata": {
        "id": "-pejvQp-0E47"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_span(start, end, shift):\n",
        "    return start+shift, end+shift"
      ],
      "metadata": {
        "id": "TN3W5_xEAtX1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_spans_flair_smarter(flair_model, some_text):\n",
        "    all_spans = []\n",
        "\n",
        "    split_text = some_text.split('\\n')\n",
        "\n",
        "    current_shift = 0\n",
        "    for line in split_text:\n",
        "        predicted_spans = []\n",
        "        sentence = Sentence(line)\n",
        "        # predict NER tags\n",
        "        flair_model.predict(sentence)\n",
        "\n",
        "        # iterate over entities\n",
        "        for entity in sentence.get_spans('ner'):\n",
        "            entity_type = entity.tag\n",
        "            entity_string = entity.text\n",
        "            if entity_type == 'PER':\n",
        "                span = calculate_span(entity.start_position, entity.end_position, current_shift)\n",
        "                predicted_spans.append(span)\n",
        "        all_spans.extend(predicted_spans)\n",
        "\n",
        "        current_shift+=len(line)+1\n",
        "    return all_spans"
      ],
      "metadata": {
        "id": "I8AmYiag5Z-O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_binary_prediction_flair(input_text):\n",
        "    filelength = len(input_text)\n",
        "    #lines = openfile.readlines()\n",
        "    #spans = []\n",
        "    #for line in tqdm(lines):\n",
        "        #spans.extend(get_spans_flair(tagger, readfile))\n",
        "    spans = get_spans_flair_smarter(tagger, input_text)\n",
        "    predicted = pd.Series(np.zeros(filelength))\n",
        "    for span in spans:\n",
        "        predicted.loc[span[0]:span[1]] = 1\n",
        "    return predicted"
      ],
      "metadata": {
        "id": "v8N_vzL_0UvS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_binary_prediction_flair(\"\"\"door\n",
        "Ludwig van Bethoven\n",
        "door\n",
        "Leo Tolstoy\n",
        "door\n",
        "Max Planck\"\"\")"
      ],
      "metadata": {
        "id": "6DEOG3UX-sKG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predicted_flair = get_binary_prediction_flair(text)"
      ],
      "metadata": {
        "id": "5ll_AB7o0rx_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(predicted_flair)"
      ],
      "metadata": {
        "id": "1zXGj3FY1rrc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Measuring flair performance"
      ],
      "metadata": {
        "id": "pSz_rZIq2Hpr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "precision_score(target, predicted_flair)"
      ],
      "metadata": {
        "id": "pmJe7Bql2Hp5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "recall_score(target, predicted_flair)"
      ],
      "metadata": {
        "id": "NGEbGxNR2Hp5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f1_score(target, predicted_flair)"
      ],
      "metadata": {
        "id": "fs5DqXEd2Hp5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generic performance measurer"
      ],
      "metadata": {
        "id": "74BhZopnuDpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(target, predicted):\n",
        "    print(precision_score(target, predicted))\n",
        "    print(recall_score(target, predicted))\n",
        "    print(f1_score(target, predicted))"
      ],
      "metadata": {
        "id": "LCI3DYaCuHdK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_on_several_files(predictor, folder_with_txt, folder_with_manual_xml):\n",
        "    all_predictions = pd.Series()\n",
        "    all_targets = pd.Series()\n",
        "    for filename in os.listdir(folder_with_txt):\n",
        "        file_id = filename.replace('.txt', '')\n",
        "        filepath = os.path.join(folder_with_txt, filename)\n",
        "        with open(filepath) as readfile:\n",
        "            filetext = readfile.text()\n",
        "        predictions = predictor(filetext)\n",
        "        targets = get_targets_for_repsective_file(folder_with_manual_xml, file_id)\n",
        "        all_predictions = pd.concat((all_predictions, predictions))\n",
        "        all_target = pd.concat((all_target, targets))\n",
        "    evaluate(all_targets, all_predictions)"
      ],
      "metadata": {
        "id": "kMjHPnvNCxXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_targets_for_repsective_file(folder_with_manual_xml, file_id):\n",
        "    for filename in os.listdir(folder_with_manual_xml):\n",
        "        if filename.startswith(file_id):\n",
        "            path_to_xml = os.path.join(folder_with_manual_xml, filename)\n",
        "            get_annotation_spans_from_xml(path_to_xml)"
      ],
      "metadata": {
        "id": "RziT0VP0w7d_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "folder_with_txt = "
      ],
      "metadata": {
        "id": "kxfjpESayn2C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "metadata": {
        "id": "aYJVuLsuyqFJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate_on_several_files(get_binary_prediction_flair, )"
      ],
      "metadata": {
        "id": "U1AjGXTGyh7s"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}