persian-ner-pipeline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Persian NER Pipeline.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c549b071edb246649b166427be14c590": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "RadioButtonsModel",
          "state": {
            "_options_labels": [
              "ARMAN",
              "PEYMA",
              "ARMAN+PEYMA"
            ],
            "_view_name": "RadioButtonsView",
            "style": "IPY_MODEL_9b516cf37dad4c63983fa604e1fb6201",
            "_dom_classes": [],
            "description": "Select your model",
            "_model_name": "RadioButtonsModel",
            "index": 0,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b207651761ab40a09caee82106db0b47"
          }
        },
        "9b516cf37dad4c63983fa604e1fb6201": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b207651761ab40a09caee82106db0b47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "139e40595d0641ea94a68d82d27ff644": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "TextareaModel",
          "state": {
            "_view_name": "TextareaView",
            "style": "IPY_MODEL_2b4b75bc8f734fff9f9668f7803697e6",
            "rows": 5,
            "_dom_classes": [],
            "description": "",
            "_model_name": "TextareaModel",
            "placeholder": "Please enter you text ...",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "یکی از بزرگترین مراکز خریدی که در شهر استانبول در ترکیه سال های اخیر ساخته شده استانبول مال است. این مرکز خرید دارای ۳۵۰ فروشگاه، فودکورت، شهربازی سرپوشیده و مراکز آثار نمایشی است که از محبوبیت ویژه ای در بین علاقمندان به خرید برخوردار است. مرکز خریدی مشهور در تور استانبول که امکانات بی نظیری برای خرید و تفریح دارد. یک رستوران سه طبقه با مساحت ۱۰۰۰ مترمربع در مرکز خرید استانبول وجود دارد که توسط یک گروه سرمایه گذار اماراتی به نام آدلر برپا شده است. این رستوران در شهرهایی نظیر دبی و ابوظبی هم دارای شعبه هستند و تمامی غذاهای خاورمیانه در آن سرو می شود؛ علاوه بر این رستوران یک شهربازی به نام مویپارک هم در این مرکز خرید وجود دارد که اولین شهربازی سرپوشیده ترکیه می باشد که دارای بازی های عظیم و هتل وحشت می باشد.",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "continuous_update": true,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58509f18dde24519be3d9e86878dbe04"
          }
        },
        "2b4b75bc8f734fff9f9668f7803697e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58509f18dde24519be3d9e86878dbe04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "90%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae620d8a31214d15b3ea12343065b179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "state": {
            "_view_name": "ButtonView",
            "style": "IPY_MODEL_6c681d462d304b7293016ea1c0bbfc69",
            "_dom_classes": [],
            "description": "Send",
            "_model_name": "ButtonModel",
            "button_style": "success",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "tooltip": "Submit",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "layout": "IPY_MODEL_55dadadf77b2444aabbf1b99117b1172",
            "_model_module": "@jupyter-widgets/controls",
            "icon": ""
          }
        },
        "6c681d462d304b7293016ea1c0bbfc69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ButtonStyleModel",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "button_color": null,
            "font_weight": "",
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "55dadadf77b2444aabbf1b99117b1172": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c052df6152a3464c98dc3f3faacd145e": {
          "model_module": "@jupyter-widgets/output",
          "model_name": "OutputModel",
          "state": {
            "_view_name": "OutputView",
            "msg_id": "",
            "_dom_classes": [],
            "_model_name": "OutputModel",
            "outputs": [
              {
                "output_type": "stream",
                "metadata": {
                  "tags": []
                },
                "text": "Predicting .... [please wait!]\n",
                "stream": "stdout"
              },
              {
                "output_type": "display_data",
                "metadata": {
                  "tags": []
                },
                "text/html": "<style>\n    .ner-box {\n        direction: rtl;\n        font-size: 18px !important;\n        line-height: 20px !important;\n        margin: 0 0 15px;\n        padding: 10px;\n        text-align: justify;\n        color: #343434 !important;\n    }\n    .token, .token span {\n        display: inline-block !important;\n        padding: 2px;\n        margin: 2px 0;\n    }\n    .token.token-ner {\n        background-color: #f6cd61;\n        font-weight: bold;\n        color: #000;\n    }\n    .token.token-ner .ner-label {\n        color: #9a1f40;\n        margin: 0px 2px;\n    }\n    </style>",
                "text/plain": "<IPython.core.display.HTML object>"
              },
              {
                "output_type": "display_data",
                "metadata": {
                  "tags": []
                },
                "text/html": "<p class=\"ner-box\"><span class=\"token\">یکی</span> <span class=\"token\">از</span> <span class=\"token\">بزرگترین</span> <span class=\"token\">مراکز</span> <span class=\"token\">خریدی</span> <span class=\"token\">که</span> <span class=\"token\">در</span> <span class=\"token token-ner\">شهر<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">استانبول<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token token-ner\">ترکیه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">سالهای</span> <span class=\"token\">اخیر</span> <span class=\"token\">ساخته</span> <span class=\"token\">شده</span> <span class=\"token token-ner\">استانبول<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">مال</span> <span class=\"token\">است</span> <span class=\"token\">.</span> <span class=\"token\">این</span> <span class=\"token\">مرکز</span> <span class=\"token\">خرید</span> <span class=\"token\">دارای</span> <span class=\"token\">۳۵۰</span> <span class=\"token\">فروشگاه</span> <span class=\"token\">،</span> <span class=\"token\">فودکورت</span> <span class=\"token\">،</span> <span class=\"token\">شهربازی</span> <span class=\"token\">سرپوشیده</span> <span class=\"token\">و</span> <span class=\"token\">مراکز</span> <span class=\"token\">اثار</span> <span class=\"token\">نمایشی</span> <span class=\"token\">است</span> <span class=\"token\">که</span> <span class=\"token\">از</span> <span class=\"token\">محبوبیت</span> <span class=\"token\">ویژهای</span> <span class=\"token\">در</span> <span class=\"token\">بین</span> <span class=\"token\">علاقمندان</span> <span class=\"token\">به</span> <span class=\"token\">خرید</span> <span class=\"token\">برخوردار</span> <span class=\"token\">است</span> <span class=\"token\">.</span> <span class=\"token\">مرکز</span> <span class=\"token\">خریدی</span> <span class=\"token\">مشهور</span> <span class=\"token\">در</span> <span class=\"token token-ner\">تور<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">استانبول<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">که</span> <span class=\"token\">امکانات</span> <span class=\"token\">بی</span> <span class=\"token\">نظیری</span> <span class=\"token\">برای</span> <span class=\"token\">خرید</span> <span class=\"token\">و</span> <span class=\"token\">تفریح</span> <span class=\"token\">دارد</span> <span class=\"token\">.</span> <span class=\"token\">یک</span> <span class=\"token\">رستوران</span> <span class=\"token\">سه</span> <span class=\"token\">طبقه</span> <span class=\"token\">با</span> <span class=\"token\">مساحت</span> <span class=\"token\">۱۰۰۰</span> <span class=\"token\">مترمربع</span> <span class=\"token\">در</span> <span class=\"token token-ner\">مرکز<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">خرید<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">استانبول<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">وجود</span> <span class=\"token\">دارد</span> <span class=\"token\">که</span> <span class=\"token\">توسط</span> <span class=\"token\">یک</span> <span class=\"token\">گروه</span> <span class=\"token\">سرمایه</span> <span class=\"token\">گذار</span> <span class=\"token token-ner\">اماراتی<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">به</span> <span class=\"token\">نام</span> <span class=\"token token-ner\">ادلر<span class=\"ner-label\">شخص</span></span> <span class=\"token\">برپا</span> <span class=\"token\">شده</span> <span class=\"token\">است</span> <span class=\"token\">.</span> <span class=\"token\">این</span> <span class=\"token\">رستوران</span> <span class=\"token\">در</span> <span class=\"token\">شهرهایی</span> <span class=\"token\">نظیر</span> <span class=\"token token-ner\">دبی<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">و</span> <span class=\"token token-ner\">ابوظبی<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">هم</span> <span class=\"token\">دارای</span> <span class=\"token\">شعبه</span> <span class=\"token\">هستند</span> <span class=\"token\">و</span> <span class=\"token\">تمامی</span> <span class=\"token\">غذاهای</span> <span class=\"token token-ner\">خاورمیانه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token\">ان</span> <span class=\"token\">سرو</span> <span class=\"token\">میشود</span> <span class=\"token\">؛</span> <span class=\"token\">علاوه</span> <span class=\"token\">بر</span> <span class=\"token\">این</span> <span class=\"token\">رستوران</span> <span class=\"token\">یک</span> <span class=\"token\">شهربازی</span> <span class=\"token\">به</span> <span class=\"token\">نام</span> <span class=\"token token-ner\">موی<span class=\"ner-label\">امکانات</span></span> <span class=\"token token-ner\">##پارک<span class=\"ner-label\">امکانات</span></span> <span class=\"token\">هم</span> <span class=\"token\">در</span> <span class=\"token\">این</span> <span class=\"token\">مرکز</span> <span class=\"token\">خرید</span> <span class=\"token\">وجود</span> <span class=\"token\">دارد</span> <span class=\"token\">که</span> <span class=\"token\">اولین</span> <span class=\"token\">شهربازی</span> <span class=\"token\">سرپوشیده</span> <span class=\"token token-ner\">ترکیه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">میباشد</span> <span class=\"token\">که</span> <span class=\"token\">دارای</span> <span class=\"token\">بازیهای</span> <span class=\"token\">عظیم</span> <span class=\"token\">و</span> <span class=\"token\">هتل</span> <span class=\"token\">وحشت</span> <span class=\"token\">میباشد</span> <span class=\"token\">.</span></p>",
                "text/plain": "<IPython.core.display.HTML object>"
              }
            ],
            "_view_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_view_count": null,
            "_view_module_version": "1.0.0",
            "layout": "IPY_MODEL_f91d0480b9fe4d99a8a31bd9f83b53c2",
            "_model_module": "@jupyter-widgets/output"
          }
        },
        "f91d0480b9fe4d99a8a31bd9f83b53c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-3mrXDugEV61",
        "colab_type": "text"
      },
      "source": [
        "# Persian NER [ARMAN, PEYMA, COMPOSITE]\n",
        "\n",
        "This task aims to extract named entities in the text, such as names and label with appropriate NER classes such as locations, organizations, etc. The datasets used for this task contain sentences that are marked with IOB format. In this format, tokens that are not part of an entity are tagged as ”O,” the ”B” tag corresponds to the first word of an object, and the ”I” tag corresponds to the rest of the terms of the same entity. Both ”B” and ”I” tags are followed by a hyphen (or underscore), followed by the entity category. Therefore, the NER task is a multi-class token classification problem that labels the tokens upon being fed a raw text. There are two primary datasets used in Persian NER, ARMAN, and PEYMA. In ParsBERT, we prepared ner for both datasets as well as a combination of both datasets.\n",
        "\n",
        "\n",
        "## ARMAN:\n",
        "ARMAN dataset holds 7,682 sentences with 250,015 sentences tagged over six different classes.\n",
        "\n",
        "1. Organization\n",
        "2. Location\n",
        "3. Facility\n",
        "4. Event\n",
        "5. Product\n",
        "6. Person\n",
        "\n",
        "\n",
        "|     Label    |   #   |\n",
        "|:------------:|:-----:|\n",
        "| Organization | 30108 |\n",
        "|   Location   | 12924 |\n",
        "|   Facility   |  4458 |\n",
        "|     Event    |  7557 |\n",
        "|    Product   |  4389 |\n",
        "|    Person    | 15645 |\n",
        "\n",
        "**Download**\n",
        "You can download the dataset from [here](https://github.com/HaniehP/PersianNER)\n",
        "\n",
        "\n",
        "\n",
        "## PEYMA:\n",
        "PEYMA dataset includes 7,145 sentences with a total of 302,530 tokens from which 41,148 tokens are tagged with seven different classes: \n",
        "\n",
        "- Organization\n",
        "- Money\n",
        "- Location\n",
        "- Date\n",
        "- Time\n",
        "- Person\n",
        "- Percent\n",
        "\n",
        "|     Label    |   #   |\n",
        "|:------------:|:-----:|\n",
        "| Organization | 16964 |\n",
        "|     Money    |  2037 |\n",
        "|   Location   |  8782 |\n",
        "|     Date     |  4259 |\n",
        "|     Time     |  732  |\n",
        "|    Person    |  7675 |\n",
        "|    Percent   |  699  |\n",
        "\n",
        "**Download**\n",
        "You can download the dataset from [here](http://nsurl.org/tasks/task-7-named-entity-recognition-ner-for-farsi/)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iDicmyYNBfnE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "try:\n",
        "    # %tensorflow_version only exists in Colab.\n",
        "    %tensorflow_version 2.x\n",
        "except Exception:\n",
        "    pass\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6sYyZvLmCYQu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q transformers\n",
        "!pip install -qU hazm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jDe1hQhCY2s",
        "colab_type": "code",
        "outputId": "dbf06a82-6dd7-458b-d65c-883ea53809b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import hazm\n",
        "\n",
        "import transformers \n",
        "from transformers import AutoTokenizer, AutoConfig\n",
        "from transformers import TFAutoModelForTokenClassification\n",
        "\n",
        "import os\n",
        "from IPython.display import display, HTML, clear_output\n",
        "from ipywidgets import widgets, Layout\n",
        "\n",
        "print()\n",
        "print('tensorflow', tf.__version__)\n",
        "print('transformers', transformers.__version__)\n",
        "print('numpy', np.__version__)\n",
        "print('pandas', pd.__version__)\n",
        "print()\n",
        "\n",
        "if tf.test.gpu_device_name() != '/device:GPU:0':\n",
        "    print()\n",
        "    print('WARNING: GPU device not found.')\n",
        "else:\n",
        "    print()\n",
        "    print('SUCCESS: Found GPU: {}'.format(tf.test.gpu_device_name()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "tensorflow 2.2.0\n",
            "transformers 2.10.0\n",
            "numpy 1.18.4\n",
            "pandas 1.0.3\n",
            "\n",
            "\n",
            "WARNING: GPU device not found.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QAsnTqSPEaJ9",
        "colab_type": "text"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8B-i_vMAEbpH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "texts = [\n",
        "    \"مدیرکل محیط زیست استان البرز با بیان اینکه با بیان اینکه موضوع شیرابه‌های زباله‌های انتقال یافته در منطقه حلقه دره خطری برای این استان است، گفت: در این مورد گزارشاتی در ۲۵ مرداد ۱۳۹۷ تقدیم مدیران استان شده است.\",\n",
        "    \"به گزارش خبرگزاری تسنیم از کرج، حسین محمدی در نشست خبری مشترک با معاون خدمات شهری شهرداری کرج که با حضور مدیرعامل سازمان‌های پسماند، پارک‌ها و فضای سبز و نماینده منابع طبیعی در سالن کنفرانس شهرداری کرج برگزار شد، اظهار داشت: ۸۰٪  جمعیت استان البرز در کلانشهر کرج زندگی می‌کنند.\",\n",
        "    \"وی افزود: با همکاری‌های مشترک بین اداره کل محیط زیست و شهرداری کرج برنامه‌های مشترکی برای حفاظت از محیط زیست در شهر کرج در دستور کار قرار گرفته که این اقدامات آثار مثبتی داشته و تاکنون نزدیک به ۱۰۰ میلیارد هزینه جهت خریداری اکس-ریس صورت گرفته است.\",\n",
        "]\n",
        "\n",
        "peyma_translate = {\n",
        "    \"B_DAT\": \"تاریخ\",\n",
        "    \"B_LOC\": \"موقعیت\",\n",
        "    \"B_MON\": \"پول\",\n",
        "    \"B_ORG\": \"سازمنان\",\n",
        "    \"B_PCT\": \"درصد\",\n",
        "    \"B_PER\": \"شخص\",\n",
        "    \"B_TIM\": \"زمان\",\n",
        "    \"I_DAT\": \"تاریخ\",\n",
        "    \"I_LOC\": \"موقعیت\",\n",
        "    \"I_MON\": \"پول\",\n",
        "    \"I_ORG\": \"سازمان\",\n",
        "    \"I_PCT\": \"درصد\",\n",
        "    \"I_PER\": \"شخص\",\n",
        "    \"I_TIM\": \"زمان\",\n",
        "    \"O\": None,\n",
        "}\n",
        "arman_translate = {\n",
        "    \"B-event\": \"رویداد\",\n",
        "    \"B-fac\": \"امکانات\",\n",
        "    \"B-loc\": \"موقعیت\",\n",
        "    \"B-org\": \"سازمان\",\n",
        "    \"B-pers\": \"شخص\",\n",
        "    \"B-pro\": \"محصول\",\n",
        "    \"I-event\": \"رویداد\",\n",
        "    \"I-fac\": \"امکانات\",\n",
        "    \"I-loc\": \"موقعیت\",\n",
        "    \"I-org\": \"سازمان\",\n",
        "    \"I-pers\": \"شخص\",\n",
        "    \"I-pro\": \"محصول\",\n",
        "    \"O\": None\n",
        "}\n",
        "\n",
        "ner_translate = {\n",
        "    \"B-date\": \"تاریخ\",\n",
        "    \"B-event\": \"رویداد\",\n",
        "    \"B-facility\": \"امکانات\",\n",
        "    \"B-location\": \"موقعیت\",\n",
        "    \"B-money\": \"پول\",\n",
        "    \"B-organization\": \"سازمان\",\n",
        "    \"B-person\": \"شخص\",\n",
        "    \"B-product\": \"محصول\",\n",
        "    \"B-time\": \"زمان\",\n",
        "    \"B-percent\": \"درصد\",\n",
        "    \"I-date\": \"تاریخ\",\n",
        "    \"I-event\": \"رویداد\",\n",
        "    \"I-facility\": \"امکانات\",\n",
        "    \"I-location\": \"موقعیت\",\n",
        "    \"I-money\": \"پول\",\n",
        "    \"I-organization\": \"سازمان\",\n",
        "    \"I-person\": \"شخص\",\n",
        "    \"I-product\": \"محصول\",\n",
        "    \"I-time\": \"زمان\",\n",
        "    \"I-percent\": \"درصد\",\n",
        "    \"O\": None\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4uMVXKVkGeOZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "normalizer = hazm.Normalizer()\n",
        "\n",
        "\n",
        "def cleanize(text):\n",
        "    \"\"\"A way to normalize and even clean the text\"\"\"\n",
        "    # clean text\n",
        "    # do some fns\n",
        "    return normalizer.normalize(text)\n",
        "\n",
        "\n",
        "def parsbert_ner_load_model(model_name):\n",
        "    \"\"\"Load the model\"\"\"\n",
        "    try:\n",
        "        config = AutoConfig.from_pretrained(model_name)\n",
        "        tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "        model = TFAutoModelForTokenClassification.from_pretrained(model_name)\n",
        "        labels = list(config.label2id.keys())\n",
        "\n",
        "        return model, tokenizer, labels\n",
        "    except:\n",
        "        return [None] * 3\n",
        "\n",
        "def parsbert_ner(texts, model_name, label_translate, visualize=True):\n",
        "    \"\"\"Predict and visualize the NER!\"\"\"\n",
        "    global css_is_load\n",
        "    \n",
        "    css_is_load = False\n",
        "    css = \"\"\"<style>\n",
        "    .ner-box {\n",
        "        direction: rtl;\n",
        "        font-size: 18px !important;\n",
        "        line-height: 20px !important;\n",
        "        margin: 0 0 15px;\n",
        "        padding: 10px;\n",
        "        text-align: justify;\n",
        "        color: #343434 !important;\n",
        "    }\n",
        "    .token, .token span {\n",
        "        display: inline-block !important;\n",
        "        padding: 2px;\n",
        "        margin: 2px 0;\n",
        "    }\n",
        "    .token.token-ner {\n",
        "        background-color: #f6cd61;\n",
        "        font-weight: bold;\n",
        "        color: #000;\n",
        "    }\n",
        "    .token.token-ner .ner-label {\n",
        "        color: #9a1f40;\n",
        "        margin: 0px 2px;\n",
        "    }\n",
        "    </style>\"\"\"\n",
        "\n",
        "    if not css_is_load:\n",
        "        display(HTML(css))\n",
        "        css_is_load = True\n",
        "\n",
        "    model, tokenizer, labels = parsbert_ner_load_model(model_name)\n",
        "\n",
        "    if not model or not tokenizer or not labels:\n",
        "        return 'Something wrong has been happened!'\n",
        "    \n",
        "    output_predictions = []\n",
        "    for sequence in texts:\n",
        "        sequence = cleanize(sequence)\n",
        "        tokens = tokenizer.tokenize(tokenizer.decode(tokenizer.encode(sequence)))\n",
        "        inputs = tokenizer.encode(sequence, return_tensors=\"tf\")\n",
        "        outputs = model(inputs)[0]\n",
        "        predictions = tf.argmax(outputs, axis=2)\n",
        "        predictions = [(token, label_translate[labels[prediction]]) for token, prediction in zip(tokens, predictions[0].numpy())]\n",
        "        \n",
        "        if not visualize:\n",
        "            output_predictions.append(predictions)\n",
        "        else:\n",
        "            pred_sequence = []\n",
        "            for token, label in predictions:\n",
        "                if token not in ['[CLS]', '[SEP]']:\n",
        "                    if label:\n",
        "                        pred_sequence.append(\n",
        "                            '<span class=\"token token-ner\">%s<span class=\"ner-label\">%s</span></span>' \n",
        "                            % (token, label))\n",
        "                    else:\n",
        "                        pred_sequence.append(\n",
        "                            '<span class=\"token\">%s</span>' \n",
        "                            % token)\n",
        "                \n",
        "            html = '<p class=\"ner-box\">%s</p>' % ' '.join(pred_sequence) \n",
        "            display(HTML(html))\n",
        "\n",
        "    return output_predictions"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aTzAnnfID8JJ",
        "colab_type": "text"
      },
      "source": [
        "# Persian ARMAN NER"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y7Zra-neHsqn",
        "colab_type": "code",
        "outputId": "8c810b6d-7da7-402c-e0af-7fa28b727cd8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 410
        }
      },
      "source": [
        "model_name = 'HooshvareLab/bert-base-parsbert-armanner-uncased'\n",
        "_ = parsbert_ner(texts, model_name, arman_translate, visualize=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>\n",
              "    .ner-box {\n",
              "        direction: rtl;\n",
              "        font-size: 18px !important;\n",
              "        line-height: 20px !important;\n",
              "        margin: 0 0 15px;\n",
              "        padding: 10px;\n",
              "        text-align: justify;\n",
              "        color: #343434 !important;\n",
              "    }\n",
              "    .token, .token span {\n",
              "        display: inline-block !important;\n",
              "        padding: 2px;\n",
              "        margin: 2px 0;\n",
              "    }\n",
              "    .token.token-ner {\n",
              "        background-color: #f6cd61;\n",
              "        font-weight: bold;\n",
              "        color: #000;\n",
              "    }\n",
              "    .token.token-ner .ner-label {\n",
              "        color: #9a1f40;\n",
              "        margin: 0px 2px;\n",
              "    }\n",
              "    </style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">مدیرکل</span> <span class=\"token token-ner\">محیط<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">زیست<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">استان<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">البرز<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">با</span> <span class=\"token\">بیان</span> <span class=\"token\">اینکه</span> <span class=\"token\">با</span> <span class=\"token\">بیان</span> <span class=\"token\">اینکه</span> <span class=\"token\">موضوع</span> <span class=\"token\">شیرابه</span> <span class=\"token\">##های</span> <span class=\"token\">زبالههای</span> <span class=\"token\">انتقال</span> <span class=\"token\">یافته</span> <span class=\"token\">در</span> <span class=\"token\">منطقه</span> <span class=\"token token-ner\">حلقه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">دره</span> <span class=\"token\">خطری</span> <span class=\"token\">برای</span> <span class=\"token\">این</span> <span class=\"token\">استان</span> <span class=\"token\">است</span> <span class=\"token\">،</span> <span class=\"token\">گفت</span> <span class=\"token\">:</span> <span class=\"token\">در</span> <span class=\"token\">این</span> <span class=\"token\">مورد</span> <span class=\"token\">گزارشاتی</span> <span class=\"token\">در</span> <span class=\"token\">۲۵</span> <span class=\"token\">مرداد</span> <span class=\"token\">۱۳۹۷</span> <span class=\"token\">تقدیم</span> <span class=\"token\">مدیران</span> <span class=\"token\">استان</span> <span class=\"token\">شده</span> <span class=\"token\">است</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">به</span> <span class=\"token\">گزارش</span> <span class=\"token token-ner\">خبرگزاری<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">تسنیم<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">از</span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">،</span> <span class=\"token token-ner\">حسین<span class=\"ner-label\">شخص</span></span> <span class=\"token token-ner\">محمدی<span class=\"ner-label\">شخص</span></span> <span class=\"token\">در</span> <span class=\"token\">نشست</span> <span class=\"token\">خبری</span> <span class=\"token\">مشترک</span> <span class=\"token\">با</span> <span class=\"token\">معاون</span> <span class=\"token\">خدمات</span> <span class=\"token\">شهری</span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">که</span> <span class=\"token\">با</span> <span class=\"token\">حضور</span> <span class=\"token\">مدیرعامل</span> <span class=\"token\">سازمانهای</span> <span class=\"token token-ner\">پسماند<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">،</span> <span class=\"token token-ner\">پارکها<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">و</span> <span class=\"token\">فضای</span> <span class=\"token\">سبز</span> <span class=\"token\">و</span> <span class=\"token\">نماینده</span> <span class=\"token\">منابع</span> <span class=\"token\">طبیعی</span> <span class=\"token\">در</span> <span class=\"token token-ner\">سالن<span class=\"ner-label\">امکانات</span></span> <span class=\"token token-ner\">کنفرانس<span class=\"ner-label\">امکانات</span></span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">امکانات</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">امکانات</span></span> <span class=\"token\">برگزار</span> <span class=\"token\">شد</span> <span class=\"token\">،</span> <span class=\"token\">اظهار</span> <span class=\"token\">داشت</span> <span class=\"token\">:</span> <span class=\"token\">۸۰</span> <span class=\"token\">٪</span> <span class=\"token\">جمعیت</span> <span class=\"token token-ner\">استان<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">البرز<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token token-ner\">کلانشهر<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">زندگی</span> <span class=\"token\">میکنند</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">وی</span> <span class=\"token\">افزود</span> <span class=\"token\">:</span> <span class=\"token\">با</span> <span class=\"token\">همکاریهای</span> <span class=\"token\">مشترک</span> <span class=\"token\">بین</span> <span class=\"token token-ner\">اداره<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">کل<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">محیط<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">زیست<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">و</span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">برنامههای</span> <span class=\"token\">مشترکی</span> <span class=\"token\">برای</span> <span class=\"token\">حفاظت</span> <span class=\"token\">از</span> <span class=\"token\">محیط</span> <span class=\"token\">زیست</span> <span class=\"token\">در</span> <span class=\"token token-ner\">شهر<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token\">دستور</span> <span class=\"token\">کار</span> <span class=\"token\">قرار</span> <span class=\"token\">گرفته</span> <span class=\"token\">که</span> <span class=\"token\">این</span> <span class=\"token\">اقدامات</span> <span class=\"token\">اثار</span> <span class=\"token\">مثبتی</span> <span class=\"token\">داشته</span> <span class=\"token\">و</span> <span class=\"token\">تاکنون</span> <span class=\"token\">نزدیک</span> <span class=\"token\">به</span> <span class=\"token\">۱۰۰</span> <span class=\"token\">میلیارد</span> <span class=\"token\">هزینه</span> <span class=\"token\">جهت</span> <span class=\"token\">خریداری</span> <span class=\"token token-ner\">اکس<span class=\"ner-label\">محصول</span></span> <span class=\"token token-ner\">-<span class=\"ner-label\">محصول</span></span> <span class=\"token token-ner\">ریس<span class=\"ner-label\">محصول</span></span> <span class=\"token\">صورت</span> <span class=\"token\">گرفته</span> <span class=\"token\">است</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fth6hzX5IG1h",
        "colab_type": "text"
      },
      "source": [
        "# Persian PEYMA NER"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jUm3HqjSGNlU",
        "colab_type": "code",
        "outputId": "ad614285-d066-4755-b2ad-07724c583767",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        }
      },
      "source": [
        "model_name = 'HooshvareLab/bert-base-parsbert-peymaner-uncased'\n",
        "_ = parsbert_ner(texts, model_name, peyma_translate, visualize=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>\n",
              "    .ner-box {\n",
              "        direction: rtl;\n",
              "        font-size: 18px !important;\n",
              "        line-height: 20px !important;\n",
              "        margin: 0 0 15px;\n",
              "        padding: 10px;\n",
              "        text-align: justify;\n",
              "        color: #343434 !important;\n",
              "    }\n",
              "    .token, .token span {\n",
              "        display: inline-block !important;\n",
              "        padding: 2px;\n",
              "        margin: 2px 0;\n",
              "    }\n",
              "    .token.token-ner {\n",
              "        background-color: #f6cd61;\n",
              "        font-weight: bold;\n",
              "        color: #000;\n",
              "    }\n",
              "    .token.token-ner .ner-label {\n",
              "        color: #9a1f40;\n",
              "        margin: 0px 2px;\n",
              "    }\n",
              "    </style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">مدیرکل</span> <span class=\"token token-ner\">محیط<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">زیست<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">استان<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">البرز<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">با</span> <span class=\"token\">بیان</span> <span class=\"token\">اینکه</span> <span class=\"token\">با</span> <span class=\"token\">بیان</span> <span class=\"token\">اینکه</span> <span class=\"token\">موضوع</span> <span class=\"token\">شیرابه</span> <span class=\"token\">##های</span> <span class=\"token\">زبالههای</span> <span class=\"token\">انتقال</span> <span class=\"token\">یافته</span> <span class=\"token\">در</span> <span class=\"token token-ner\">منطقه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">حلقه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">دره<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">خطری</span> <span class=\"token\">برای</span> <span class=\"token\">این</span> <span class=\"token\">استان</span> <span class=\"token\">است</span> <span class=\"token\">،</span> <span class=\"token\">گفت</span> <span class=\"token\">:</span> <span class=\"token\">در</span> <span class=\"token\">این</span> <span class=\"token\">مورد</span> <span class=\"token\">گزارشاتی</span> <span class=\"token\">در</span> <span class=\"token token-ner\">۲۵<span class=\"ner-label\">تاریخ</span></span> <span class=\"token token-ner\">مرداد<span class=\"ner-label\">تاریخ</span></span> <span class=\"token token-ner\">۱۳۹۷<span class=\"ner-label\">تاریخ</span></span> <span class=\"token\">تقدیم</span> <span class=\"token\">مدیران</span> <span class=\"token\">استان</span> <span class=\"token\">شده</span> <span class=\"token\">است</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">به</span> <span class=\"token\">گزارش</span> <span class=\"token token-ner\">خبرگزاری<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">تسنیم<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">از</span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">،</span> <span class=\"token token-ner\">حسین<span class=\"ner-label\">شخص</span></span> <span class=\"token token-ner\">محمدی<span class=\"ner-label\">شخص</span></span> <span class=\"token\">در</span> <span class=\"token\">نشست</span> <span class=\"token\">خبری</span> <span class=\"token\">مشترک</span> <span class=\"token\">با</span> <span class=\"token\">معاون</span> <span class=\"token\">خدمات</span> <span class=\"token\">شهری</span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">که</span> <span class=\"token\">با</span> <span class=\"token\">حضور</span> <span class=\"token\">مدیرعامل</span> <span class=\"token token-ner\">سازمانهای<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">پسماند<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">،<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">پارکها<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">و<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">فضای<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">سبز<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">و</span> <span class=\"token\">نماینده</span> <span class=\"token token-ner\">منابع<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">طبیعی<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">در</span> <span class=\"token token-ner\">سالن<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کنفرانس<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">برگزار</span> <span class=\"token\">شد</span> <span class=\"token\">،</span> <span class=\"token\">اظهار</span> <span class=\"token\">داشت</span> <span class=\"token\">:</span> <span class=\"token\">۸۰</span> <span class=\"token\">٪</span> <span class=\"token\">جمعیت</span> <span class=\"token token-ner\">استان<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">البرز<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token token-ner\">کلانشهر<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">زندگی</span> <span class=\"token\">میکنند</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">وی</span> <span class=\"token\">افزود</span> <span class=\"token\">:</span> <span class=\"token\">با</span> <span class=\"token\">همکاریهای</span> <span class=\"token\">مشترک</span> <span class=\"token\">بین</span> <span class=\"token token-ner\">اداره<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">کل<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">محیط<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">زیست<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">و</span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">سازمنان</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">برنامههای</span> <span class=\"token\">مشترکی</span> <span class=\"token\">برای</span> <span class=\"token\">حفاظت</span> <span class=\"token\">از</span> <span class=\"token\">محیط</span> <span class=\"token\">زیست</span> <span class=\"token\">در</span> <span class=\"token token-ner\">شهر<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token\">دستور</span> <span class=\"token\">کار</span> <span class=\"token\">قرار</span> <span class=\"token\">گرفته</span> <span class=\"token\">که</span> <span class=\"token\">این</span> <span class=\"token\">اقدامات</span> <span class=\"token\">اثار</span> <span class=\"token\">مثبتی</span> <span class=\"token\">داشته</span> <span class=\"token\">و</span> <span class=\"token\">تاکنون</span> <span class=\"token\">نزدیک</span> <span class=\"token\">به</span> <span class=\"token\">۱۰۰</span> <span class=\"token token-ner\">میلیارد<span class=\"ner-label\">پول</span></span> <span class=\"token\">هزینه</span> <span class=\"token\">جهت</span> <span class=\"token\">خریداری</span> <span class=\"token\">اکس</span> <span class=\"token\">-</span> <span class=\"token\">ریس</span> <span class=\"token\">صورت</span> <span class=\"token\">گرفته</span> <span class=\"token\">است</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "83zSbY4kINkP",
        "colab_type": "text"
      },
      "source": [
        "# Persian ParsBERT NER"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rjwPppJ4IM7b",
        "colab_type": "code",
        "outputId": "2bceeb00-5e01-49ed-c557-69528c1e5a03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        }
      },
      "source": [
        "model_name = 'HooshvareLab/bert-base-parsbert-ner-uncased'\n",
        "_ = parsbert_ner(texts, model_name, ner_translate, visualize=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>\n",
              "    .ner-box {\n",
              "        direction: rtl;\n",
              "        font-size: 18px !important;\n",
              "        line-height: 20px !important;\n",
              "        margin: 0 0 15px;\n",
              "        padding: 10px;\n",
              "        text-align: justify;\n",
              "        color: #343434 !important;\n",
              "    }\n",
              "    .token, .token span {\n",
              "        display: inline-block !important;\n",
              "        padding: 2px;\n",
              "        margin: 2px 0;\n",
              "    }\n",
              "    .token.token-ner {\n",
              "        background-color: #f6cd61;\n",
              "        font-weight: bold;\n",
              "        color: #000;\n",
              "    }\n",
              "    .token.token-ner .ner-label {\n",
              "        color: #9a1f40;\n",
              "        margin: 0px 2px;\n",
              "    }\n",
              "    </style>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">مدیرکل</span> <span class=\"token token-ner\">محیط<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">زیست<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">استان<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">البرز<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">با</span> <span class=\"token\">بیان</span> <span class=\"token\">اینکه</span> <span class=\"token\">با</span> <span class=\"token\">بیان</span> <span class=\"token\">اینکه</span> <span class=\"token\">موضوع</span> <span class=\"token\">شیرابه</span> <span class=\"token\">##های</span> <span class=\"token\">زبالههای</span> <span class=\"token\">انتقال</span> <span class=\"token\">یافته</span> <span class=\"token\">در</span> <span class=\"token token-ner\">منطقه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">حلقه<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">دره<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">خطری</span> <span class=\"token\">برای</span> <span class=\"token\">این</span> <span class=\"token\">استان</span> <span class=\"token\">است</span> <span class=\"token\">،</span> <span class=\"token\">گفت</span> <span class=\"token\">:</span> <span class=\"token\">در</span> <span class=\"token\">این</span> <span class=\"token\">مورد</span> <span class=\"token\">گزارشاتی</span> <span class=\"token\">در</span> <span class=\"token token-ner\">۲۵<span class=\"ner-label\">تاریخ</span></span> <span class=\"token token-ner\">مرداد<span class=\"ner-label\">تاریخ</span></span> <span class=\"token token-ner\">۱۳۹۷<span class=\"ner-label\">تاریخ</span></span> <span class=\"token\">تقدیم</span> <span class=\"token\">مدیران</span> <span class=\"token\">استان</span> <span class=\"token\">شده</span> <span class=\"token\">است</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">به</span> <span class=\"token\">گزارش</span> <span class=\"token token-ner\">خبرگزاری<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">تسنیم<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">از</span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">،</span> <span class=\"token token-ner\">حسین<span class=\"ner-label\">شخص</span></span> <span class=\"token token-ner\">محمدی<span class=\"ner-label\">شخص</span></span> <span class=\"token\">در</span> <span class=\"token\">نشست</span> <span class=\"token\">خبری</span> <span class=\"token\">مشترک</span> <span class=\"token\">با</span> <span class=\"token\">معاون</span> <span class=\"token\">خدمات</span> <span class=\"token\">شهری</span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">که</span> <span class=\"token\">با</span> <span class=\"token\">حضور</span> <span class=\"token\">مدیرعامل</span> <span class=\"token token-ner\">سازمانهای<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">پسماند<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">،<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">پارکها<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">و<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">فضای<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">سبز<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">و</span> <span class=\"token\">نماینده</span> <span class=\"token token-ner\">منابع<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">طبیعی<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">در</span> <span class=\"token token-ner\">سالن<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کنفرانس<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">برگزار</span> <span class=\"token\">شد</span> <span class=\"token\">،</span> <span class=\"token\">اظهار</span> <span class=\"token\">داشت</span> <span class=\"token\">:</span> <span class=\"token token-ner\">۸۰<span class=\"ner-label\">درصد</span></span> <span class=\"token token-ner\">٪<span class=\"ner-label\">درصد</span></span> <span class=\"token\">جمعیت</span> <span class=\"token token-ner\">استان<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">البرز<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token\">کلانشهر</span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">زندگی</span> <span class=\"token\">میکنند</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p class=\"ner-box\"><span class=\"token\">وی</span> <span class=\"token\">افزود</span> <span class=\"token\">:</span> <span class=\"token\">با</span> <span class=\"token\">همکاریهای</span> <span class=\"token\">مشترک</span> <span class=\"token\">بین</span> <span class=\"token token-ner\">اداره<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">کل<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">محیط<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">زیست<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">و</span> <span class=\"token token-ner\">شهرداری<span class=\"ner-label\">سازمان</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">سازمان</span></span> <span class=\"token\">برنامههای</span> <span class=\"token\">مشترکی</span> <span class=\"token\">برای</span> <span class=\"token\">حفاظت</span> <span class=\"token\">از</span> <span class=\"token\">محیط</span> <span class=\"token\">زیست</span> <span class=\"token\">در</span> <span class=\"token token-ner\">شهر<span class=\"ner-label\">موقعیت</span></span> <span class=\"token token-ner\">کرج<span class=\"ner-label\">موقعیت</span></span> <span class=\"token\">در</span> <span class=\"token\">دستور</span> <span class=\"token\">کار</span> <span class=\"token\">قرار</span> <span class=\"token\">گرفته</span> <span class=\"token\">که</span> <span class=\"token\">این</span> <span class=\"token\">اقدامات</span> <span class=\"token\">اثار</span> <span class=\"token\">مثبتی</span> <span class=\"token\">داشته</span> <span class=\"token\">و</span> <span class=\"token\">تاکنون</span> <span class=\"token\">نزدیک</span> <span class=\"token\">به</span> <span class=\"token token-ner\">۱۰۰<span class=\"ner-label\">پول</span></span> <span class=\"token token-ner\">میلیارد<span class=\"ner-label\">پول</span></span> <span class=\"token\">هزینه</span> <span class=\"token\">جهت</span> <span class=\"token\">خریداری</span> <span class=\"token\">اکس</span> <span class=\"token\">-</span> <span class=\"token\">ریس</span> <span class=\"token\">صورت</span> <span class=\"token\">گرفته</span> <span class=\"token\">است</span> <span class=\"token\">.</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0MWiqaGIJcHc",
        "colab_type": "code",
        "outputId": "b4c6e5bc-b556-42a3-b143-b81e4f030cfe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 610,
          "referenced_widgets": [
            "c549b071edb246649b166427be14c590",
            "9b516cf37dad4c63983fa604e1fb6201",
            "b207651761ab40a09caee82106db0b47",
            "139e40595d0641ea94a68d82d27ff644",
            "2b4b75bc8f734fff9f9668f7803697e6",
            "58509f18dde24519be3d9e86878dbe04",
            "ae620d8a31214d15b3ea12343065b179",
            "6c681d462d304b7293016ea1c0bbfc69",
            "55dadadf77b2444aabbf1b99117b1172",
            "c052df6152a3464c98dc3f3faacd145e",
            "f91d0480b9fe4d99a8a31bd9f83b53c2"
          ]
        }
      },
      "source": [
        "#@title Live Playground { display-mode: \"form\" }\n",
        "\n",
        "submit_wd = widgets.Button(description='Send', disabled=False, button_style='success', tooltip='Submit')\n",
        "text_wd = widgets.Textarea(placeholder='Please enter you text ...', rows=5, layout=Layout(width='90%'))\n",
        "\n",
        "config_wd = widgets.RadioButtons(options=['ARMAN', 'PEYMA', 'ARMAN+PEYMA'], description='Select your model', disabled=False)\n",
        "\n",
        "output_wd = widgets.Output()\n",
        "\n",
        "display(HTML(\"\"\"\n",
        "<h2>Persian NER <small>[ARMAN, PEYMA, ARMAN+PEYMA]</small></h2>\n",
        "<p style=\"padding: 2px 20px; margin: 0 0 20px;\">\n",
        "</p>\n",
        "<br /><br />\n",
        "\"\"\"))\n",
        "\n",
        "display(config_wd)\n",
        "display(text_wd)\n",
        "display(submit_wd)\n",
        "display(output_wd)\n",
        "\n",
        "configs = {\n",
        "    'ARMAN': [\n",
        "        'HooshvareLab/bert-base-parsbert-armanner-uncased',\n",
        "        arman_translate\n",
        "    ], \n",
        "    'PEYMA': [\n",
        "        'HooshvareLab/bert-base-parsbert-peymaner-uncased',\n",
        "        peyma_translate\n",
        "    ],\n",
        "    'ARMAN+PEYMA': [\n",
        "        'HooshvareLab/bert-base-parsbert-ner-uncased',\n",
        "        ner_translate\n",
        "    ]\n",
        "}\n",
        "\n",
        "def submit_text(sender):\n",
        "    with output_wd:\n",
        "        clear_output(wait=True)\n",
        "        text = text_wd.value\n",
        "        config = configs[config_wd.value]\n",
        "        print('Predicting .... [please wait!]')\n",
        "        _ = parsbert_ner([text], config[0], config[1], visualize=True)\n",
        "\n",
        "\n",
        "submit_wd.on_click(submit_text)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "<h2>Persian NER <small>[ARMAN, PEYMA, ARMAN+PEYMA]</small></h2>\n",
              "<p style=\"padding: 2px 20px; margin: 0 0 20px;\">\n",
              "</p>\n",
              "<br /><br />\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c549b071edb246649b166427be14c590",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "RadioButtons(description='Select your model', options=('ARMAN', 'PEYMA', 'ARMAN+PEYMA'), value='ARMAN')"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "139e40595d0641ea94a68d82d27ff644",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Textarea(value='', layout=Layout(width='90%'), placeholder='Please enter you text ...', rows=5)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ae620d8a31214d15b3ea12343065b179",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Button(button_style='success', description='Send', style=ButtonStyle(), tooltip='Submit')"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c052df6152a3464c98dc3f3faacd145e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Output()"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SnbDx48Ig70",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}