benchmark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ner-benchmark.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "RTg7QMrgOzh0"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "740a990cb09140e79886b4afb305dde6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8867af95026e4c048a3503a565477eeb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6f5bfd2fb2ba417eb0429fb48b735112",
              "IPY_MODEL_f2c883cec9544bd7a35adcc2f58e4764"
            ]
          }
        },
        "8867af95026e4c048a3503a565477eeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f5bfd2fb2ba417eb0429fb48b735112": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_076f7ce5b7b94d4fbbafce1008cd83dd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1572,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1572,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5bcdf38330c48a088fb924f321eaf29"
          }
        },
        "f2c883cec9544bd7a35adcc2f58e4764": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_84b6933d7fde4b1db19f7b773cce2a36",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.57k/1.57k [00:05&lt;00:00, 288B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_75fb2c64430a42138907763af018127e"
          }
        },
        "076f7ce5b7b94d4fbbafce1008cd83dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5bcdf38330c48a088fb924f321eaf29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "84b6933d7fde4b1db19f7b773cce2a36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "75fb2c64430a42138907763af018127e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "062ad4150ca54156b4dcf90e0991aef9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d9149f0b672448589fd5ab47d1db032b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_928676766e2443b6b7c8dabc1c8957df",
              "IPY_MODEL_ee4143debdad40779adcc0ed6c71a0e4"
            ]
          }
        },
        "d9149f0b672448589fd5ab47d1db032b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "928676766e2443b6b7c8dabc1c8957df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_26a0b058956b49c18f36e503c8c89906",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 857476,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 857476,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_221d4d144fea44db9f02e935389c9fb6"
          }
        },
        "ee4143debdad40779adcc0ed6c71a0e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_90f09e872db44524ad420577e29cbf4b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 857k/857k [00:00&lt;00:00, 1.16MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b735b6a55e74c61a27084e023cb514a"
          }
        },
        "26a0b058956b49c18f36e503c8c89906": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "221d4d144fea44db9f02e935389c9fb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "90f09e872db44524ad420577e29cbf4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b735b6a55e74c61a27084e023cb514a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "38adfcd9dcc44f23a4ee98a0a2405ff9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b4f6a44340b34560a773129706a89c1e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3ee1c7ad1021482c9912d66ef9ad6637",
              "IPY_MODEL_b21fcf1b2c9d46b58a1eb317a3fd152f"
            ]
          }
        },
        "b4f6a44340b34560a773129706a89c1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ee1c7ad1021482c9912d66ef9ad6637": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_876286bf35584c3db9820fd0690a000c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3229336,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3229336,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c0e51f50612d4fcf94cb898d2c4116fe"
          }
        },
        "b21fcf1b2c9d46b58a1eb317a3fd152f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_260cd720471d4ba59382319e990455d2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.23M/3.23M [00:04&lt;00:00, 750kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d84304e26ae8483eb1ddf6db43529631"
          }
        },
        "876286bf35584c3db9820fd0690a000c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c0e51f50612d4fcf94cb898d2c4116fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "260cd720471d4ba59382319e990455d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d84304e26ae8483eb1ddf6db43529631": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "223bf9f3a2a248f2a95f1a4d5b281a82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c2190fc617774933989cda2d31a41eab",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_843f0f9ca535498c86e20f3c476ac4d6",
              "IPY_MODEL_7fb0f5757f7c4b5194b44d33a536808a"
            ]
          }
        },
        "c2190fc617774933989cda2d31a41eab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "843f0f9ca535498c86e20f3c476ac4d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4f3a92da5e58450cbe02b4c1ecef3fe3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 186,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 186,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b80eb194e2364533885857b5971397ca"
          }
        },
        "7fb0f5757f7c4b5194b44d33a536808a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6b16d40e967a47dc9eb8b0a64c537605",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 186/186 [00:01&lt;00:00, 170B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_835ee161441841f69458e7ae27ec1b1e"
          }
        },
        "4f3a92da5e58450cbe02b4c1ecef3fe3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b80eb194e2364533885857b5971397ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b16d40e967a47dc9eb8b0a64c537605": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "835ee161441841f69458e7ae27ec1b1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e5fc997facf41ec9301a0272891c0aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0dd8362bb39417994dccdf0bf4209b0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3c31655782794204a5f948eae8562195",
              "IPY_MODEL_f02a80a2737e4fc6b409aa35c658dec5"
            ]
          }
        },
        "b0dd8362bb39417994dccdf0bf4209b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c31655782794204a5f948eae8562195": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a288b3cfbab345d9949076215427d92c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 499,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 499,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_484dd7dbb340436696813bebb6fa7c88"
          }
        },
        "f02a80a2737e4fc6b409aa35c658dec5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_469c0fe631aa475da2350272c7403254",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 499/499 [00:00&lt;00:00, 1.06kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_27ebb972859f4f59bc28d1f2803501d3"
          }
        },
        "a288b3cfbab345d9949076215427d92c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "484dd7dbb340436696813bebb6fa7c88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "469c0fe631aa475da2350272c7403254": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "27ebb972859f4f59bc28d1f2803501d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8315bd581e0d49f5ba193ab8f9fb2dbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4afdc58abb454b408524e482cc566446",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2d56ec1e4a40401dab715c15cceabecb",
              "IPY_MODEL_0fe3844cbcba40c0a36d76997299f29e"
            ]
          }
        },
        "4afdc58abb454b408524e482cc566446": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d56ec1e4a40401dab715c15cceabecb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_95277d62192d48ffafb0f713cc739677",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 44451929,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 44451929,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cf857c52a61947cea95f4a2940f6d104"
          }
        },
        "0fe3844cbcba40c0a36d76997299f29e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8145895b2d0f4a479f839f65ea01fa15",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 44.5M/44.5M [00:01&lt;00:00, 33.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1a35dc523ee84f2493d69c40cc758b7b"
          }
        },
        "95277d62192d48ffafb0f713cc739677": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cf857c52a61947cea95f4a2940f6d104": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8145895b2d0f4a479f839f65ea01fa15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1a35dc523ee84f2493d69c40cc758b7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "731ec5d4ac0e4cfea66258b3d99bec52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_81d0d46918e84d5885866849f680d9f5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d06e6a3094f44896ac6c9ebfcbe062aa",
              "IPY_MODEL_2f97f24a5fac487586a542682307765c"
            ]
          }
        },
        "81d0d46918e84d5885866849f680d9f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d06e6a3094f44896ac6c9ebfcbe062aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d4f18e17aebd4a728781843b1ee36c7d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1357,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1357,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_be18f78fc71f47f2ad2e84d6747ecde6"
          }
        },
        "2f97f24a5fac487586a542682307765c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_16626393ccdf4eccbba056650e2d0cbb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.36k/1.36k [00:00&lt;00:00, 1.99kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d69faeb806694e139a1b204ad2e24cbf"
          }
        },
        "d4f18e17aebd4a728781843b1ee36c7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "be18f78fc71f47f2ad2e84d6747ecde6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16626393ccdf4eccbba056650e2d0cbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d69faeb806694e139a1b204ad2e24cbf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3cfe533414f433686f91e5322b2d2ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6436abe5211b461d8efa2eb2028f364c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f5e35d72bbaf45d3b79509b37d75552f",
              "IPY_MODEL_97992c39b9054490ba9478d9d8f6cab9"
            ]
          }
        },
        "6436abe5211b461d8efa2eb2028f364c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5e35d72bbaf45d3b79509b37d75552f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e51245b565364f4883a3f081609a7e17",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_234e3e547be441e99cfae1c01f7bcee0"
          }
        },
        "97992c39b9054490ba9478d9d8f6cab9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_deaef04510d2450287e77c807b3a159e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:02&lt;00:00, 489kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a5c9f8b96e74fc5abc91bf1d69dcfa5"
          }
        },
        "e51245b565364f4883a3f081609a7e17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "234e3e547be441e99cfae1c01f7bcee0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "deaef04510d2450287e77c807b3a159e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a5c9f8b96e74fc5abc91bf1d69dcfa5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4a4420f538c42538f2e545269f9b6cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c220ab2004d64c8e84d26703c86d43f2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a3fe177fffd24e989754fb05af17639f",
              "IPY_MODEL_d4b72fb5d798433e8c74c25e6405186a"
            ]
          }
        },
        "c220ab2004d64c8e84d26703c86d43f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a3fe177fffd24e989754fb05af17639f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c165cacfebbf4cf696f7df313ea2b1bb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1653272ac5ce44329bea5b5a64bc1407"
          }
        },
        "d4b72fb5d798433e8c74c25e6405186a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_55cb3114c0e848ee9f22bcee1f134155",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 135B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6e9d9eb588aa4a5faf106f8fb354d3a0"
          }
        },
        "c165cacfebbf4cf696f7df313ea2b1bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1653272ac5ce44329bea5b5a64bc1407": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55cb3114c0e848ee9f22bcee1f134155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6e9d9eb588aa4a5faf106f8fb354d3a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "32f9c9df1efd49c7b306d7cc983195a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dde6b01fbf504ea785ebc18312769f58",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2f115a31a45e4ed5b031682045a812d0",
              "IPY_MODEL_681d83516082405b9cff230a124d223a"
            ]
          }
        },
        "dde6b01fbf504ea785ebc18312769f58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f115a31a45e4ed5b031682045a812d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b866c6ff72324b97bee39985430e0b5a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b078c027972c43f68c5ec1ca3c1def5e"
          }
        },
        "681d83516082405b9cff230a124d223a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f164c78f53b744919992bce4bae6edf7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.00/2.00 [00:00&lt;00:00, 10.2B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a51a74f4461b43cdb4a9ee6937981ddc"
          }
        },
        "b866c6ff72324b97bee39985430e0b5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b078c027972c43f68c5ec1ca3c1def5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f164c78f53b744919992bce4bae6edf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a51a74f4461b43cdb4a9ee6937981ddc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6685c1af24754a9b8afb1dde1389009a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c4ac27e658e047be849e85a5bced10ee",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_87528c0af63243b19e973150d86b30e3",
              "IPY_MODEL_b9faff595b5e4d20819a7f42704c0067"
            ]
          }
        },
        "c4ac27e658e047be849e85a5bced10ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87528c0af63243b19e973150d86b30e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b33bb1ec47bd443090d77402e463be48",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 651478185,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 651478185,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_76f703b6be394f6fbaa9e5bc9132f4ae"
          }
        },
        "b9faff595b5e4d20819a7f42704c0067": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a728cccb6f3a464b8ed2d5a18ea0f66b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 651M/651M [00:15&lt;00:00, 42.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6ffe46e435b349f8a8e7833b7153860f"
          }
        },
        "b33bb1ec47bd443090d77402e463be48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "76f703b6be394f6fbaa9e5bc9132f4ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a728cccb6f3a464b8ed2d5a18ea0f66b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6ffe46e435b349f8a8e7833b7153860f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7301d5b2784248dfae506426a58863b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b4ef81263c8c4472be8521ecfaae4dcf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ae6d12be652749d594cb4148618c90f4",
              "IPY_MODEL_ebdd97ba670d45d8acf68e0c552625fa"
            ]
          }
        },
        "b4ef81263c8c4472be8521ecfaae4dcf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae6d12be652749d594cb4148618c90f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_22cbc589b23e43a08cb8a70f64bec7bf",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 699,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 699,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_14713245725f4090a2cde4ca592366a3"
          }
        },
        "ebdd97ba670d45d8acf68e0c552625fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7c21c204c0f8483bb218e374da2715e6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 699/699 [00:00&lt;00:00, 950B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac67fc675e284cba94d6cf0b11b686b1"
          }
        },
        "22cbc589b23e43a08cb8a70f64bec7bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "14713245725f4090a2cde4ca592366a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7c21c204c0f8483bb218e374da2715e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ac67fc675e284cba94d6cf0b11b686b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc957ec21b4f4cea81dba07c8c204eef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6ea013abe9224f73bd3c61a07f195006",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bb4536cbbf084e6d91fab5f347d84f6a",
              "IPY_MODEL_55c90f639f004efbad7d796694bacc31"
            ]
          }
        },
        "6ea013abe9224f73bd3c61a07f195006": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb4536cbbf084e6d91fab5f347d84f6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9670dc9d89af430d848bb0464cdc55a2",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5069051,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5069051,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fcefda86bdd64c9a9843bc9a1ce97df2"
          }
        },
        "55c90f639f004efbad7d796694bacc31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_852b7646a53644a3874626a6f8a18d7d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5.07M/5.07M [00:03&lt;00:00, 1.46MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_baa3c022504b4fc1a27a91b849597df0"
          }
        },
        "9670dc9d89af430d848bb0464cdc55a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fcefda86bdd64c9a9843bc9a1ce97df2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "852b7646a53644a3874626a6f8a18d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "baa3c022504b4fc1a27a91b849597df0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b46e6e326dc442f49f534be4fd636461": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6612f18c66af42378c4f8e7a2d34e239",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_baf8bfb4787c48ffa29d76a4fcc3ec5d",
              "IPY_MODEL_7873f4a59467498b9e52a70e789a5f10"
            ]
          }
        },
        "6612f18c66af42378c4f8e7a2d34e239": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "baf8bfb4787c48ffa29d76a4fcc3ec5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fa5163b1c23b4c258b7dc15bfbffed85",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 150,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 150,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4bc29d42c8dd4f7284fccc21831328bf"
          }
        },
        "7873f4a59467498b9e52a70e789a5f10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_80d9fb8b44034efda2eb9aab27fa6adc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 150/150 [00:01&lt;00:00, 91.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c51dc16fc80d40d888a65e54105fd3a8"
          }
        },
        "fa5163b1c23b4c258b7dc15bfbffed85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4bc29d42c8dd4f7284fccc21831328bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80d9fb8b44034efda2eb9aab27fa6adc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c51dc16fc80d40d888a65e54105fd3a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c308d2d892d1425bb26ef6a8b4514149": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_20ffea64882b40f79bcaba662bc61f11",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8f1346003b0c41459ac4c0410ee68333",
              "IPY_MODEL_f35472be819545aba946fbbd89e5dbbb"
            ]
          }
        },
        "20ffea64882b40f79bcaba662bc61f11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f1346003b0c41459ac4c0410ee68333": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1a4b8ad9a80148f8941486d3aa0714e6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de6e9a4c19db44f19064dcd2eff7537f"
          }
        },
        "f35472be819545aba946fbbd89e5dbbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cf08174a61b648d28d2c545a30f09b55",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.00/2.00 [00:00&lt;00:00, 2.02B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4aa938d0d02a4e049b29be0d344e3e5e"
          }
        },
        "1a4b8ad9a80148f8941486d3aa0714e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de6e9a4c19db44f19064dcd2eff7537f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf08174a61b648d28d2c545a30f09b55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4aa938d0d02a4e049b29be0d344e3e5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9dd8d0d3e8ab4daabba3d2f271763c53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cf26895f2f49427091e88bcfd40019a4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_86926ebed1c048d683acb4a260c56c8d",
              "IPY_MODEL_f866b3f8cc4844919b0106af498ab7d4"
            ]
          }
        },
        "cf26895f2f49427091e88bcfd40019a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "86926ebed1c048d683acb4a260c56c8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b8806af543c34003bee7f86f416b4d37",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1112459008,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1112459008,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07d34f4737ff44a399904262a629cd2b"
          }
        },
        "f866b3f8cc4844919b0106af498ab7d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1bf6ed64658e42038a9665aa74251783",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.11G/1.11G [00:24&lt;00:00, 45.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d9c5c25f7464fb38754f8463a739ca9"
          }
        },
        "b8806af543c34003bee7f86f416b4d37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07d34f4737ff44a399904262a629cd2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1bf6ed64658e42038a9665aa74251783": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d9c5c25f7464fb38754f8463a739ca9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9d0b2696074b46879f6f8132faa6b8ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7d23502f535543bfbbdbe0f685f6dfa1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_061636e8fc294d4ba57ff15477a97a72",
              "IPY_MODEL_5de62407aead4332a7c871c2afb65fa2"
            ]
          }
        },
        "7d23502f535543bfbbdbe0f685f6dfa1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "061636e8fc294d4ba57ff15477a97a72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9348835bd95a40c39472f2b54dc26ddb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1124,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1124,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80d23c4a39524334956020a30cd23b44"
          }
        },
        "5de62407aead4332a7c871c2afb65fa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ecd1fb04eb7a4bb6937268d1a66fcbb3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.12k/1.12k [00:00&lt;00:00, 4.51kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_645773f0eabb4dd49d27f30362181d09"
          }
        },
        "9348835bd95a40c39472f2b54dc26ddb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "80d23c4a39524334956020a30cd23b44": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ecd1fb04eb7a4bb6937268d1a66fcbb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "645773f0eabb4dd49d27f30362181d09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93f73fabe300408183d8fc387a187981": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bd7422b62304400cad68767f3de43e8d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a134ab2d1820490a9c67ab3580236c79",
              "IPY_MODEL_2f46ae8c30774cbd8854a66407498530"
            ]
          }
        },
        "bd7422b62304400cad68767f3de43e8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a134ab2d1820490a9c67ab3580236c79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c9a96717370d498e84bded7e7f6f1e53",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a1e902506782420984948151fea41ce4"
          }
        },
        "2f46ae8c30774cbd8854a66407498530": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e8d8701f8bbf4556983a410384b695e4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:04&lt;00:00, 270kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d73533926084e99adea6e65f8276e46"
          }
        },
        "c9a96717370d498e84bded7e7f6f1e53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a1e902506782420984948151fea41ce4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e8d8701f8bbf4556983a410384b695e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d73533926084e99adea6e65f8276e46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b64322e3ffc49e2b54bb60d39dec20f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_209333e1db8148468d5982306950dc57",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f32144cfd095404c9122cbbd25559ce1",
              "IPY_MODEL_45ffee06ed2e472d8af41b9a4455ce6b"
            ]
          }
        },
        "209333e1db8148468d5982306950dc57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f32144cfd095404c9122cbbd25559ce1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a19d00d6f672415682e2f3e3d259dcd6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_20907867e0994ca8a222d52621690347"
          }
        },
        "45ffee06ed2e472d8af41b9a4455ce6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_74a0f24dfaef4d5b8ae1ff7fd629ab60",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:01&lt;00:00, 93.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_974cf4dd7c89413f956b2112b9628006"
          }
        },
        "a19d00d6f672415682e2f3e3d259dcd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "20907867e0994ca8a222d52621690347": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74a0f24dfaef4d5b8ae1ff7fd629ab60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "974cf4dd7c89413f956b2112b9628006": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "19c90bda858443dcbae7f6ba576039bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b5b7b968348040bd86516dabbd087edc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9217de938fad43a4ac385618ee069391",
              "IPY_MODEL_81d15bda83db4da4a4d3504c1f3f9106"
            ]
          }
        },
        "b5b7b968348040bd86516dabbd087edc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9217de938fad43a4ac385618ee069391": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_845c625486fc415c97974b645e1884f6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 354,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 354,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b35590632a1d40069f17233af292579a"
          }
        },
        "81d15bda83db4da4a4d3504c1f3f9106": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_179b35e60d364fd1a1c5cd775ca47583",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 354/354 [00:00&lt;00:00, 1.63kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6470a038a522450ab8151c98fa72f47b"
          }
        },
        "845c625486fc415c97974b645e1884f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b35590632a1d40069f17233af292579a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "179b35e60d364fd1a1c5cd775ca47583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6470a038a522450ab8151c98fa72f47b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "03e590ff0ce54c3bb70497b98a343777": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_09662437a6af4643bd046400efca62c5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b88be82d8dd04678a86fa740e1ad17fc",
              "IPY_MODEL_bde2258c8bb04769a7d2b3832dd6ecea"
            ]
          }
        },
        "09662437a6af4643bd046400efca62c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b88be82d8dd04678a86fa740e1ad17fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a3fc61cfa9e742208c9d9221238ca0f8",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 649311328,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 649311328,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5be4c8b7c7ca40e2b78c6b5db1331c53"
          }
        },
        "bde2258c8bb04769a7d2b3832dd6ecea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7dac4f5a990a4c18b3b6b1499ae3d855",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 649M/649M [00:37&lt;00:00, 17.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b4161a8e30ea40c0a0088235aad3af60"
          }
        },
        "a3fc61cfa9e742208c9d9221238ca0f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5be4c8b7c7ca40e2b78c6b5db1331c53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7dac4f5a990a4c18b3b6b1499ae3d855": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b4161a8e30ea40c0a0088235aad3af60": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cc8ae7f518b847e199b601717debd49c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_671dbf7ab46a43da96f80a325d5df9de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_14aecdd86ae2440da90dd9e604fbf599",
              "IPY_MODEL_2b981119248b4f838643be99a2bed3d0"
            ]
          }
        },
        "671dbf7ab46a43da96f80a325d5df9de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "14aecdd86ae2440da90dd9e604fbf599": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_479e907d7c14401a85ba89345c6dde13",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1124,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1124,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f636d79955674c2b89c1a243987fc71e"
          }
        },
        "2b981119248b4f838643be99a2bed3d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a8fb34c2c0884eb58285666cda1279cc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.12k/1.12k [00:00&lt;00:00, 6.19kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_26488cebb72e47f3ba1bd8fbabb12fad"
          }
        },
        "479e907d7c14401a85ba89345c6dde13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f636d79955674c2b89c1a243987fc71e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a8fb34c2c0884eb58285666cda1279cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "26488cebb72e47f3ba1bd8fbabb12fad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9dd5593e8a99449d8d3fcaed7052bfc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_94b2c4839a964a2b9b9313545ba93a82",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d186cd5f37fe49b3a568ea81a81b7bad",
              "IPY_MODEL_253068a8570e48efa263cc14848ec8c0"
            ]
          }
        },
        "94b2c4839a964a2b9b9313545ba93a82": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d186cd5f37fe49b3a568ea81a81b7bad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a883103d05f844ae95cd3ef2b504bcf8",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_92438dc593ad464780696aeeb6a2c55d"
          }
        },
        "253068a8570e48efa263cc14848ec8c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_55159a4c579e4ea09bc8ce5ae704ebf2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:00&lt;00:00, 1.67MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_45aa8fb090b741baa05b1433b943c73a"
          }
        },
        "a883103d05f844ae95cd3ef2b504bcf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "92438dc593ad464780696aeeb6a2c55d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55159a4c579e4ea09bc8ce5ae704ebf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "45aa8fb090b741baa05b1433b943c73a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e2213f76bb74c09b228641f2a25d105": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_14eda72734cc47eda9f2215760f4c5a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2cc642ba0da5482fb9adaefa266404ba",
              "IPY_MODEL_f6a2f7d1dd5041439e2ff97d1e83e354"
            ]
          }
        },
        "14eda72734cc47eda9f2215760f4c5a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2cc642ba0da5482fb9adaefa266404ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e2e115907bf646edb88eb4613eb225f3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a082ddb6bdfe48fbac4bd9449707a044"
          }
        },
        "f6a2f7d1dd5041439e2ff97d1e83e354": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f76786ad2ea34d5cacfd7e4042135d18",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 214B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ec8f2a7bd224364bfe80e93b62f8b59"
          }
        },
        "e2e115907bf646edb88eb4613eb225f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a082ddb6bdfe48fbac4bd9449707a044": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f76786ad2ea34d5cacfd7e4042135d18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ec8f2a7bd224364bfe80e93b62f8b59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "367de1b57efd48608c647a04dac8e34d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0ba6367897a241b4b31980077f442a1d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_209f15203ddb48139c047ddfbe013def",
              "IPY_MODEL_41b22596ecfb4d1fa50fe8a14502c85c"
            ]
          }
        },
        "0ba6367897a241b4b31980077f442a1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "209f15203ddb48139c047ddfbe013def": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_824212d44eb1483b9d8843bbd6d1999c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 354,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 354,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3350d0511e7846fc8978b6bbe4567d1b"
          }
        },
        "41b22596ecfb4d1fa50fe8a14502c85c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c1f566b5e5db4e7f96b64d7a67a7a91a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 354/354 [00:00&lt;00:00, 1.87kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0058a66157584b6698434b5481d06fc6"
          }
        },
        "824212d44eb1483b9d8843bbd6d1999c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3350d0511e7846fc8978b6bbe4567d1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c1f566b5e5db4e7f96b64d7a67a7a91a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0058a66157584b6698434b5481d06fc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a4aa8a86b88416492cd7cc14b1aeab8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5db5d64647d94e7fa0c4a0a4f7d700af",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1f16ffc3782d442bafda0c6c42f59a7a",
              "IPY_MODEL_372eca8f78fd4e52a15ef0bb73d0182f"
            ]
          }
        },
        "5db5d64647d94e7fa0c4a0a4f7d700af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f16ffc3782d442bafda0c6c42f59a7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e11d45a97f784ff6a88ca708b9a09bf9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 649311328,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 649311328,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42db72165339459c8ffae5390ba03f0c"
          }
        },
        "372eca8f78fd4e52a15ef0bb73d0182f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1085049fb01e4098bf681d26d49b59f2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 649M/649M [00:15&lt;00:00, 42.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5bea16934e16416aba618cf38d4f50b5"
          }
        },
        "e11d45a97f784ff6a88ca708b9a09bf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42db72165339459c8ffae5390ba03f0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1085049fb01e4098bf681d26d49b59f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5bea16934e16416aba618cf38d4f50b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97a4ba89598941159c9f960c2749b70e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b214fe27f48d420f9d4220599d9b65e3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_007e48cdc9fe47cfb9959d021112ddeb",
              "IPY_MODEL_0b62ccb625034afc96f3ae6542bc4498"
            ]
          }
        },
        "b214fe27f48d420f9d4220599d9b65e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "007e48cdc9fe47cfb9959d021112ddeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5373d106e3af44f98f77dcd6a7b4348a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1196,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1196,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9d1f2f62a57e43fd8b728550758977e1"
          }
        },
        "0b62ccb625034afc96f3ae6542bc4498": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c84821460e7a40af8b75134e5acf4099",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.20k/1.20k [00:01&lt;00:00, 800B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aac9baaad7ef48ee91fd97f7033a0ab0"
          }
        },
        "5373d106e3af44f98f77dcd6a7b4348a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9d1f2f62a57e43fd8b728550758977e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c84821460e7a40af8b75134e5acf4099": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aac9baaad7ef48ee91fd97f7033a0ab0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10ebfcfa5ea442f4b15fce3c4920a255": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1f4447651aeb4f4581a397a125f27875",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_da2bd89fcb194b108b10e3e0484cfceb",
              "IPY_MODEL_2f938ebf96d84f52ba9e52be9964d976"
            ]
          }
        },
        "1f4447651aeb4f4581a397a125f27875": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da2bd89fcb194b108b10e3e0484cfceb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_12ef8cfc846d477e9305d3d639dac9a0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b57b949607474828ad3c0fa3d17d07f3"
          }
        },
        "2f938ebf96d84f52ba9e52be9964d976": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6d2ffe05c15445518b3904ed5cc20469",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:00&lt;00:00, 1.24MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce8de109632c444b9fd407310963cc55"
          }
        },
        "12ef8cfc846d477e9305d3d639dac9a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b57b949607474828ad3c0fa3d17d07f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6d2ffe05c15445518b3904ed5cc20469": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce8de109632c444b9fd407310963cc55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2dafb342404a46f0b6fd87cf6382d37c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1b7293a8d4f8404fa51bd46a11c5f511",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8bae6e70fe844fc2bee4a157d64a3c39",
              "IPY_MODEL_172ba28dcef54e46850fa44f47608a7a"
            ]
          }
        },
        "1b7293a8d4f8404fa51bd46a11c5f511": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8bae6e70fe844fc2bee4a157d64a3c39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2b3a05bb91dd4737990565873884f179",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0397fbdef27f477fabe0093727e9cede"
          }
        },
        "172ba28dcef54e46850fa44f47608a7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a23d6e99ab5e4a2ba845ab40a8b1498b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 356B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_673c90fe0cf24416ac4fa1a5a8d30092"
          }
        },
        "2b3a05bb91dd4737990565873884f179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0397fbdef27f477fabe0093727e9cede": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a23d6e99ab5e4a2ba845ab40a8b1498b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "673c90fe0cf24416ac4fa1a5a8d30092": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a5be964f98546aabdb68b253c88e573": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c49abd18d56a47b693f547b72b46e3ab",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_121121c4fe5a40e08a637fec951e9eba",
              "IPY_MODEL_260f96dc84ee48d4890bbf3060c2f9dd"
            ]
          }
        },
        "c49abd18d56a47b693f547b72b46e3ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "121121c4fe5a40e08a637fec951e9eba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_72355c9281c04e10b342cf25132dcde3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 354,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 354,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1687efa68df0407092a61fd0d7a0c0e3"
          }
        },
        "260f96dc84ee48d4890bbf3060c2f9dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0cf052f4ffd04302ac209fde39f254d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 354/354 [00:00&lt;00:00, 1.66kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ea1e969254f04cbf905fda3458eb4376"
          }
        },
        "72355c9281c04e10b342cf25132dcde3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1687efa68df0407092a61fd0d7a0c0e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0cf052f4ffd04302ac209fde39f254d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ea1e969254f04cbf905fda3458eb4376": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c36bc42b5a3a49b0b94c9ced194525c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_32988af100894ea1a7880ec516507c07",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7d27ee0586104b8e86556a54e79e6c60",
              "IPY_MODEL_3df100c50b5c4c149772cbe7afc9fb4e"
            ]
          }
        },
        "32988af100894ea1a7880ec516507c07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d27ee0586104b8e86556a54e79e6c60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_db29108d394e4093a7cc4f673dd1d065",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 649317472,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 649317472,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c6547c32f8a441f5bb7836c505c64ba9"
          }
        },
        "3df100c50b5c4c149772cbe7afc9fb4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_50ca024a96ee4edd8845c31d52cbc745",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 649M/649M [00:19&lt;00:00, 34.1MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c38018121a8422b93c387380af63b1b"
          }
        },
        "db29108d394e4093a7cc4f673dd1d065": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c6547c32f8a441f5bb7836c505c64ba9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50ca024a96ee4edd8845c31d52cbc745": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c38018121a8422b93c387380af63b1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f1d669af757410fbca1d82de36ee6d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f29b1d66c94446b88161d3eaa860a7af",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca06c75378974a2e96295c0138d5096f",
              "IPY_MODEL_87cb6c0ff37e48b8b8ea04817a82f4a1"
            ]
          }
        },
        "f29b1d66c94446b88161d3eaa860a7af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca06c75378974a2e96295c0138d5096f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_981b5832a12142d1bc8a9fda7e01482c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c7273e6e8fc4d32ab2c8212b1b222f2"
          }
        },
        "87cb6c0ff37e48b8b8ea04817a82f4a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f3b8ed0006df4f998d14fa9f9937e6df",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [00:00&lt;00:00, 5.00kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_82e826dd3af44669b05c0554159bbdb5"
          }
        },
        "981b5832a12142d1bc8a9fda7e01482c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c7273e6e8fc4d32ab2c8212b1b222f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3b8ed0006df4f998d14fa9f9937e6df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "82e826dd3af44669b05c0554159bbdb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16bc9f6b41da497f9a8615af4f746fd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a3890ea815e640e2a2ae0fef295a5eb0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d41f090bc1f344a8b2a508d2cb92f457",
              "IPY_MODEL_df23a558ba51482db899018bd43270b4"
            ]
          }
        },
        "a3890ea815e640e2a2ae0fef295a5eb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d41f090bc1f344a8b2a508d2cb92f457": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c9d747aeb992474992a2b32c32ea23eb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c11a461d82e472990ebba1bc9a298b5"
          }
        },
        "df23a558ba51482db899018bd43270b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bf5fd737b3824372af891910953228dc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:01&lt;00:00, 1.01MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f083bd1d45a4edaa63628a6a8c528db"
          }
        },
        "c9d747aeb992474992a2b32c32ea23eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c11a461d82e472990ebba1bc9a298b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bf5fd737b3824372af891910953228dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f083bd1d45a4edaa63628a6a8c528db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a1ee17260b4493ca12780c7580a5e8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fa0103e1b80e44b1824fc0729d32a46c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6291961738354e6298fc3189fc19ed8a",
              "IPY_MODEL_691fd430de6046108ac06ec1318d8632"
            ]
          }
        },
        "fa0103e1b80e44b1824fc0729d32a46c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6291961738354e6298fc3189fc19ed8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d3493eac3ff64868b49f2991cb0750c8",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dce1064e787140239f4988baf02c151d"
          }
        },
        "691fd430de6046108ac06ec1318d8632": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c35a378cf38f4e2c84b0ba44c347fe3a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 219B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fab34d6581ff49fd9fbdd2cc7519a3e3"
          }
        },
        "d3493eac3ff64868b49f2991cb0750c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dce1064e787140239f4988baf02c151d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c35a378cf38f4e2c84b0ba44c347fe3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fab34d6581ff49fd9fbdd2cc7519a3e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e97b653a23014991a3032fa9645556b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f0bedb8936b2432da1e8608ceb4d030b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d2b9a8068351447baad4040b702cd9f4",
              "IPY_MODEL_6ff66087fdc047e1bcd0ca306630938c"
            ]
          }
        },
        "f0bedb8936b2432da1e8608ceb4d030b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d2b9a8068351447baad4040b702cd9f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7b14b6ab59cd4e50bfa9ee040c6637cf",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bd3b40ee26114c739d037a42bd66a1f2"
          }
        },
        "6ff66087fdc047e1bcd0ca306630938c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7d941b34a1824f00a399ebfcff673c5b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.00/2.00 [00:00&lt;00:00, 9.39B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83bc89ff87884f83aa879fc717dd024e"
          }
        },
        "7b14b6ab59cd4e50bfa9ee040c6637cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bd3b40ee26114c739d037a42bd66a1f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d941b34a1824f00a399ebfcff673c5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83bc89ff87884f83aa879fc717dd024e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e1d7a073749d4d8abe13643e76eb10a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_593fa987743142a3ba348d8f9bb1bc90",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f7acd42c219349cea9401edc9cf04918",
              "IPY_MODEL_8d2e78f27fef4d73857be4b3547a742f"
            ]
          }
        },
        "593fa987743142a3ba348d8f9bb1bc90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7acd42c219349cea9401edc9cf04918": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_08ed57cbf4824f64bcc3293373d90a66",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 651459729,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 651459729,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_90d9f097252749c98c308dfc5c48ee72"
          }
        },
        "8d2e78f27fef4d73857be4b3547a742f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c6fa55ed77164bb490c541ed42a2429d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 651M/651M [00:20&lt;00:00, 31.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dd125644642f4db3949075538f58f225"
          }
        },
        "08ed57cbf4824f64bcc3293373d90a66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "90d9f097252749c98c308dfc5c48ee72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c6fa55ed77164bb490c541ed42a2429d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dd125644642f4db3949075538f58f225": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ddbbd5d4e7df4616ab769e4800e1dd28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3b9ef28ba713432c89bc10758df9ecaa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cec72dee43574c37b7c9ecce13f23414",
              "IPY_MODEL_e5c73a7d0772457fa3e32132280cac28"
            ]
          }
        },
        "3b9ef28ba713432c89bc10758df9ecaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cec72dee43574c37b7c9ecce13f23414": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_02983885fd0f4cab9b973b2c57e7bf32",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1124,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1124,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_55a359eba2a8477d88c4abb146c49d72"
          }
        },
        "e5c73a7d0772457fa3e32132280cac28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8a26909b53104374ba14d64f74d4e1bb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.12k/1.12k [00:01&lt;00:00, 918B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_90101cece0ab4af9b9dd87992564c95b"
          }
        },
        "02983885fd0f4cab9b973b2c57e7bf32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "55a359eba2a8477d88c4abb146c49d72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a26909b53104374ba14d64f74d4e1bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "90101cece0ab4af9b9dd87992564c95b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "21c68bd6a69c455d8dec19130ee786fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c859846604e040dc95dd3f0eba3baebc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_069064350b924e71a5b7ae92cdb4cf51",
              "IPY_MODEL_52486fe33a0c4dc4bd1f92d19eef9ecb"
            ]
          }
        },
        "c859846604e040dc95dd3f0eba3baebc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "069064350b924e71a5b7ae92cdb4cf51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2e67b8a3d5ec4408ad31458e696fdb8d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_098c562696554826ad5e036cf6508ff0"
          }
        },
        "52486fe33a0c4dc4bd1f92d19eef9ecb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8816d9f61912431698e080fe4cedf5aa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:04&lt;00:00, 244kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c06935c82cb477f9e899835bd1a75af"
          }
        },
        "2e67b8a3d5ec4408ad31458e696fdb8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "098c562696554826ad5e036cf6508ff0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8816d9f61912431698e080fe4cedf5aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c06935c82cb477f9e899835bd1a75af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61ca72ad4cda4018b0f608beb64ce7e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e68f9b165ffd4be784e7268add1af5d8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2cbb164f7bae4b7185e99100f2e35de6",
              "IPY_MODEL_2a467e95c5d9478eb4d2c5d6e4a81a28"
            ]
          }
        },
        "e68f9b165ffd4be784e7268add1af5d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2cbb164f7bae4b7185e99100f2e35de6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f2ead96607c4ebd8be732ff55e2a556",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0aadae3b6ff947548727d31420510da4"
          }
        },
        "2a467e95c5d9478eb4d2c5d6e4a81a28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5e48e974ca654aeb897bdb7906fcce5f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:01&lt;00:00, 82.7B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ddc0db2b656144109b5497d0af98c42d"
          }
        },
        "2f2ead96607c4ebd8be732ff55e2a556": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0aadae3b6ff947548727d31420510da4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e48e974ca654aeb897bdb7906fcce5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ddc0db2b656144109b5497d0af98c42d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3836e256f3b4d5eb9eaafbe56e1edfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_80e83788717d4baaabcc030c76584660",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_390e3dacef5c488b965e2a8643af2a70",
              "IPY_MODEL_0a74233ac2504b57a3bd21a5c934b9a5"
            ]
          }
        },
        "80e83788717d4baaabcc030c76584660": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "390e3dacef5c488b965e2a8643af2a70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9f4887836ac94e96ac45a89cb60305d2",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 354,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 354,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b99bc5c10603483d943c9ddf9eb20d75"
          }
        },
        "0a74233ac2504b57a3bd21a5c934b9a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_76a5ba4de18b4ac2908501fd9a41cdd3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 354/354 [00:00&lt;00:00, 2.02kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1b1986e8c5d4b14855bd26f45cf84a7"
          }
        },
        "9f4887836ac94e96ac45a89cb60305d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b99bc5c10603483d943c9ddf9eb20d75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "76a5ba4de18b4ac2908501fd9a41cdd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1b1986e8c5d4b14855bd26f45cf84a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48a90a675bff4d4c969e0a9fa5c13424": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bfc36645bb4b4d37817e84d5ebc631f9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_15def708108f431d9ccbcb1dc3103451",
              "IPY_MODEL_affbf0081d0b4243be160285bf2969ed"
            ]
          }
        },
        "bfc36645bb4b4d37817e84d5ebc631f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15def708108f431d9ccbcb1dc3103451": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cd34da28d73c47ef88e034d47866745f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 649311328,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 649311328,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7bfc840549cb43cdb1ec2c529b21697c"
          }
        },
        "affbf0081d0b4243be160285bf2969ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_944c5683177147bcb947fffef1c7e644",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 649M/649M [00:36&lt;00:00, 17.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e0f211affada407a8b61df6726d7025f"
          }
        },
        "cd34da28d73c47ef88e034d47866745f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7bfc840549cb43cdb1ec2c529b21697c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "944c5683177147bcb947fffef1c7e644": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e0f211affada407a8b61df6726d7025f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef91b2f52698484fb89650b83333d5a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b3988b24758d4d40b03567f1b5f24e7b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_67c21a70804543a7850ccf66b6f80129",
              "IPY_MODEL_4b736c7fd39c45339d850b751f053031"
            ]
          }
        },
        "b3988b24758d4d40b03567f1b5f24e7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "67c21a70804543a7850ccf66b6f80129": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_75e8389e698342efb0dcc64820d78f2b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61112889ca0542839573d5f4c565bdba"
          }
        },
        "4b736c7fd39c45339d850b751f053031": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e893da8bd2a143dabf8a03ae9eae345a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [00:04&lt;00:00, 217B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4aa624c1ecbe4f1584a3d6ac0ecdc023"
          }
        },
        "75e8389e698342efb0dcc64820d78f2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61112889ca0542839573d5f4c565bdba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e893da8bd2a143dabf8a03ae9eae345a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4aa624c1ecbe4f1584a3d6ac0ecdc023": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66befa77c3944d4e9339b713bf6ecd3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_81c0454bfc024f2ab6d3de09f9d9ef1b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4e1e6ead3ecd447e9fe1043f987667f8",
              "IPY_MODEL_54161db780ea420e8232d44200c0153a"
            ]
          }
        },
        "81c0454bfc024f2ab6d3de09f9d9ef1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e1e6ead3ecd447e9fe1043f987667f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_512627527a7e49cf8b1105eef1312437",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1215509,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1215509,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a7e787e127ea4291988595f4507a1afe"
          }
        },
        "54161db780ea420e8232d44200c0153a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fa8302519e0e422aaa0bc9d983357d9c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.22M/1.22M [00:03&lt;00:00, 358kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cc03250ff47246778f3476e0e636914b"
          }
        },
        "512627527a7e49cf8b1105eef1312437": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a7e787e127ea4291988595f4507a1afe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa8302519e0e422aaa0bc9d983357d9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cc03250ff47246778f3476e0e636914b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cb0b7472c01e46f990ff056fbd7ddd22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6f8dc8d1e0f9473e867504459144ebef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_24ab94ff8dfa4e79baf4ccc108600607",
              "IPY_MODEL_e735ba01a0ac4e84b9ac2004e18f6e8e"
            ]
          }
        },
        "6f8dc8d1e0f9473e867504459144ebef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "24ab94ff8dfa4e79baf4ccc108600607": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8df4818bf8cb45bb945614f900ef1a3e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0ac453887ca5476c9c0a2054f0fb47e0"
          }
        },
        "e735ba01a0ac4e84b9ac2004e18f6e8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_52f2edeea77f4c1aacd17e525e8c7767",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:01&lt;00:00, 66.8B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1b3aef3589c745118ee14aba4ac08e7d"
          }
        },
        "8df4818bf8cb45bb945614f900ef1a3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0ac453887ca5476c9c0a2054f0fb47e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52f2edeea77f4c1aacd17e525e8c7767": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1b3aef3589c745118ee14aba4ac08e7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93bee8a4880a4b27b670cad320556b02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5b99105196dc4116b72823d8d9566bd0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e97e1694f5484ef88c02dc60d188e30b",
              "IPY_MODEL_bae3f74bf4884321983bee5955ccf1f5"
            ]
          }
        },
        "5b99105196dc4116b72823d8d9566bd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e97e1694f5484ef88c02dc60d188e30b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f23be764e14d482b97cf2b8e3046662e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a015845b9c8a443abcfcf243959080f8"
          }
        },
        "bae3f74bf4884321983bee5955ccf1f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f7b988af55664744b7dc38fe683ed8ab",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.00/2.00 [00:00&lt;00:00, 10.1B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2271ba12f76b4c9e91684e135a2d1d72"
          }
        },
        "f23be764e14d482b97cf2b8e3046662e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a015845b9c8a443abcfcf243959080f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7b988af55664744b7dc38fe683ed8ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2271ba12f76b4c9e91684e135a2d1d72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d5bef9964ed47708973eb6e636e0c0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_291888c597f84cd78874bbe324231b93",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_337d0777f5e84d5e9af78e5bdd714a85",
              "IPY_MODEL_b10f40a1bafb4e6e816cc8ba224b5b4b"
            ]
          }
        },
        "291888c597f84cd78874bbe324231b93": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "337d0777f5e84d5e9af78e5bdd714a85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b6f9643c72364197ba8cb45dab2a8619",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 651459729,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 651459729,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0f9f269432c4a3983ff1d4d267eb4a1"
          }
        },
        "b10f40a1bafb4e6e816cc8ba224b5b4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1c5586da885b42e9b15021bf547d5446",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 651M/651M [00:54&lt;00:00, 11.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6ce6008cefdb404b8aa66d5220a8e6fd"
          }
        },
        "b6f9643c72364197ba8cb45dab2a8619": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0f9f269432c4a3983ff1d4d267eb4a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c5586da885b42e9b15021bf547d5446": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6ce6008cefdb404b8aa66d5220a8e6fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/overfit-ir/persian-twitter-ner/blob/master/benchmark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ebZsy7HYObGg"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hWNzPQINXSE1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7924891b-7cba-42be-b932-712f793022ec"
      },
      "source": [
        "!pip -q install transformers==4.2.2\n",
        "!pip -q install sentencepiece\n",
        "!pip -q install nereval"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.8MB 6.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 870kB 44.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9MB 41.4MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 4.3MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iRrA1EolVVEU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "94f6faa7-2c2a-403e-b3f9-152e0153f025"
      },
      "source": [
        "import transformers\n",
        "transformers.__version__"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'4.2.2'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nF-7m57sBgdp"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from transformers import (\n",
        "    pipeline, \n",
        "    AutoConfig, \n",
        "    AutoTokenizer, \n",
        "    AutoModel, \n",
        "    AutoModelForTokenClassification\n",
        ")\n",
        "from pprint import pprint"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1Awb1uBEExX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed20ce02-4986-4583-e908-dcc07e29fe4e"
      },
      "source": [
        "! rm -rf data\n",
        "! wget -q --show-progress https://raw.githubusercontent.com/overfit-ir/persian-twitter-ner/master/twitter_data/persian-ner-twitter-data/test.txt\n",
        "! wget -q --show-progress https://raw.githubusercontent.com/overfit-ir/persian-twitter-ner/master/twitter_data/persian-ner-twitter-data/train.txt\n",
        "! mkdir data && mv test.txt data/ && mv train.txt data/"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test.txt            100%[===================>] 108.10K  --.-KB/s    in 0.03s   \n",
            "train.txt           100%[===================>]   2.20M  --.-KB/s    in 0.1s    \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0-CkLklZYBv1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd436a52-dbd8-40da-a6c3-1ca6ee9b2a48"
      },
      "source": [
        "! rm -rf data_peyma\n",
        "! wget -q --show-progress https://raw.githubusercontent.com/overfit-ir/persian-twitter-ner/master/ner_data/peyma/test.txt\n",
        "! wget -q --show-progress https://raw.githubusercontent.com/overfit-ir/persian-twitter-ner/master/ner_data/peyma/train.txt\n",
        "! mkdir data_peyma && mv test.txt data_peyma/ && mv train.txt data_peyma/"
      ],
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test.txt            100%[===================>] 359.39K  --.-KB/s    in 0.04s   \n",
            "train.txt           100%[===================>]   2.64M  --.-KB/s    in 0.1s    \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6_zKUcGOlCU"
      },
      "source": [
        "# Convert to Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qJtW5DROJJSl"
      },
      "source": [
        "from pathlib import Path\n",
        "import re\n",
        "\n",
        "def convert_lines_to_text(file_path, separator='\\t'):\n",
        "    file_path = Path(file_path)\n",
        "\n",
        "    raw_text = file_path.read_text().strip()\n",
        "    raw_docs = re.split(r'\\n\\t?\\n', raw_text)\n",
        "    token_docs = []\n",
        "    tag_docs = []\n",
        "    for doc in raw_docs:\n",
        "        tokens = []\n",
        "        tags = []\n",
        "        for line in doc.split('\\n'):\n",
        "            token, tag = line.split(separator, 1)\n",
        "            tokens.append(token)\n",
        "            tags.append(tag)\n",
        "        token_docs.append(tokens)\n",
        "        tag_docs.append(tags)\n",
        "\n",
        "    return token_docs, tag_docs"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9Q5H5PXOqf7"
      },
      "source": [
        "texts, tags = convert_lines_to_text('data/test.txt')\n",
        "texts_train, tags_train = convert_lines_to_text('data/train.txt')"
      ],
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HYPvTHz3Ub1D",
        "outputId": "5cbe3e01-a785-44c4-90de-b6805661cbfd"
      },
      "source": [
        "s = ''\n",
        "for word in texts[8]:\n",
        "  s += word + ' '\n",
        "print(s)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "شعار ندهید همین الان بسیاری از سران نظام فعلی ایران دارای تابعیت دوگانه ایرانی و عراقی هستند برادران لاریجانی سردار نقدی دکتر صالحی حتی مرحوم شاهرودی زمانی که رییس قوه قضاییه بود فارسی را به لهجه عربی صحبت میکرد مهم تعهد و عشق به وطن است که در سایه آموزش و رضایت بدست میآید \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FumYJgSZYn8m"
      },
      "source": [
        "texts_peyma, tags_peyma = convert_lines_to_text('data_peyma/test.txt', separator='|')\n",
        "texts_peyma_train, tags_peyma_train = convert_lines_to_text('data_peyma/test.txt', separator='|')"
      ],
      "execution_count": 112,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ZH1hr2oZ4jI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6b625f50-cda0-4105-e140-a5e47ecb5cbd"
      },
      "source": [
        "s = ''\n",
        "for word in texts_peyma[0]:\n",
        "  s += word + ' '\n",
        "print(s)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "کنایه سرلشگر فیروزآبادی به پادشاه عربستان و پسرش \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RTg7QMrgOzh0"
      },
      "source": [
        "# Benchmark"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFxoYA2L_KNC"
      },
      "source": [
        "import logging\n",
        "from collections import namedtuple\n",
        "from copy import deepcopy\n",
        "\n",
        "logging.basicConfig(\n",
        "    format=\"%(asctime)s %(name)s %(levelname)s: %(message)s\",\n",
        "    datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
        "    level=\"DEBUG\",\n",
        ")\n",
        "\n",
        "Entity = namedtuple(\"Entity\", \"e_type start_offset end_offset\")\n",
        "\n",
        "class Evaluator():\n",
        "\n",
        "    def __init__(self, true, pred, tags):\n",
        "        \"\"\"\n",
        "        \"\"\"\n",
        "\n",
        "        if len(true) != len(pred):\n",
        "            raise ValueError(\"Number of predicted documents does not equal true\")\n",
        "\n",
        "        self.true = true\n",
        "        self.pred = pred\n",
        "        self.tags = tags\n",
        "\n",
        "        # Setup dict into which metrics will be stored.\n",
        "\n",
        "        self.metrics_results = {\n",
        "            'correct': 0,\n",
        "            'incorrect': 0,\n",
        "            'partial': 0,\n",
        "            'missed': 0,\n",
        "            'spurious': 0,\n",
        "            'possible': 0,\n",
        "            'actual': 0,\n",
        "            'precision': 0,\n",
        "            'recall': 0,\n",
        "        }\n",
        "\n",
        "        # Copy results dict to cover the four schemes.\n",
        "\n",
        "        self.results = {\n",
        "            'strict': deepcopy(self.metrics_results),\n",
        "            'ent_type': deepcopy(self.metrics_results),\n",
        "            'partial':deepcopy(self.metrics_results),\n",
        "            'exact':deepcopy(self.metrics_results),\n",
        "            }\n",
        "\n",
        "        # Create an accumulator to store results\n",
        "\n",
        "        self.evaluation_agg_entities_type = {e: deepcopy(self.results) for e in tags}\n",
        "\n",
        "\n",
        "    def evaluate(self):\n",
        "\n",
        "        logging.info(\n",
        "            \"Imported %s predictions for %s true examples\",\n",
        "            len(self.pred), len(self.true)\n",
        "        )\n",
        "\n",
        "        for true_ents, pred_ents in zip(self.true, self.pred):\n",
        "\n",
        "            # Check that the length of the true and predicted examples are the\n",
        "            # same. This must be checked here, because another error may not\n",
        "            # be thrown if the lengths do not match.\n",
        "\n",
        "            if len(true_ents) != len(pred_ents):\n",
        "                raise ValueError(\"Prediction length does not match true example length\")\n",
        "\n",
        "            # Compute results for one message\n",
        "\n",
        "            tmp_results, tmp_agg_results = compute_metrics(\n",
        "                collect_named_entities(true_ents),\n",
        "                collect_named_entities(pred_ents),\n",
        "                self.tags\n",
        "            )\n",
        "\n",
        "            # Cycle through each result and accumulate\n",
        "\n",
        "            # TODO: Combine these loops below:\n",
        "\n",
        "            for eval_schema in self.results:\n",
        "\n",
        "                for metric in self.results[eval_schema]:\n",
        "\n",
        "                    self.results[eval_schema][metric] += tmp_results[eval_schema][metric]\n",
        "\n",
        "            # Calculate global precision and recall\n",
        "\n",
        "            self.results = compute_precision_recall_wrapper(self.results)\n",
        "\n",
        "            # Aggregate results by entity type\n",
        "\n",
        "            for e_type in self.tags:\n",
        "\n",
        "                for eval_schema in tmp_agg_results[e_type]:\n",
        "\n",
        "                    for metric in tmp_agg_results[e_type][eval_schema]:\n",
        "\n",
        "                        self.evaluation_agg_entities_type[e_type][eval_schema][metric] += tmp_agg_results[e_type][eval_schema][metric]\n",
        "\n",
        "                # Calculate precision recall at the individual entity level\n",
        "\n",
        "                self.evaluation_agg_entities_type[e_type] = compute_precision_recall_wrapper(self.evaluation_agg_entities_type[e_type])\n",
        "\n",
        "        return self.results, self.evaluation_agg_entities_type\n",
        "\n",
        "\n",
        "def collect_named_entities(tokens):\n",
        "    \"\"\"\n",
        "    Creates a list of Entity named-tuples, storing the entity type and the start and end\n",
        "    offsets of the entity.\n",
        "\n",
        "    :param tokens: a list of tags\n",
        "    :return: a list of Entity named-tuples\n",
        "    \"\"\"\n",
        "\n",
        "    named_entities = []\n",
        "    start_offset = None\n",
        "    end_offset = None\n",
        "    ent_type = None\n",
        "\n",
        "    for offset, token_tag in enumerate(tokens):\n",
        "\n",
        "        if token_tag == 'O':\n",
        "            if ent_type is not None and start_offset is not None:\n",
        "                end_offset = offset - 1\n",
        "                named_entities.append(Entity(ent_type, start_offset, end_offset))\n",
        "                start_offset = None\n",
        "                end_offset = None\n",
        "                ent_type = None\n",
        "\n",
        "        elif ent_type is None:\n",
        "            ent_type = token_tag[2:]\n",
        "            start_offset = offset\n",
        "\n",
        "        elif ent_type != token_tag[2:] or (ent_type == token_tag[2:] and token_tag[:1] == 'B'):\n",
        "\n",
        "            end_offset = offset - 1\n",
        "            named_entities.append(Entity(ent_type, start_offset, end_offset))\n",
        "\n",
        "            # start of a new entity\n",
        "            ent_type = token_tag[2:]\n",
        "            start_offset = offset\n",
        "            end_offset = None\n",
        "\n",
        "    # catches an entity that goes up until the last token\n",
        "\n",
        "    if ent_type is not None and start_offset is not None and end_offset is None:\n",
        "        named_entities.append(Entity(ent_type, start_offset, len(tokens)-1))\n",
        "\n",
        "    return named_entities\n",
        "\n",
        "\n",
        "def compute_metrics(true_named_entities, pred_named_entities, tags):\n",
        "\n",
        "\n",
        "    eval_metrics = {'correct': 0, 'incorrect': 0, 'partial': 0, 'missed': 0, 'spurious': 0, 'precision': 0, 'recall': 0}\n",
        "\n",
        "    # overall results\n",
        "    \n",
        "    evaluation = {\n",
        "        'strict': deepcopy(eval_metrics),\n",
        "        'ent_type': deepcopy(eval_metrics),\n",
        "        'partial': deepcopy(eval_metrics),\n",
        "        'exact': deepcopy(eval_metrics)\n",
        "    }\n",
        "\n",
        "    # results by entity type\n",
        "\n",
        "    evaluation_agg_entities_type = {e: deepcopy(evaluation) for e in tags}\n",
        "\n",
        "    # keep track of entities that overlapped\n",
        "\n",
        "    true_which_overlapped_with_pred = []\n",
        "\n",
        "    # Subset into only the tags that we are interested in.\n",
        "    # NOTE: we remove the tags we don't want from both the predicted and the\n",
        "    # true entities. This covers the two cases where mismatches can occur:\n",
        "    #\n",
        "    # 1) Where the model predicts a tag that is not present in the true data\n",
        "    # 2) Where there is a tag in the true data that the model is not capable of\n",
        "    # predicting.\n",
        "\n",
        "    true_named_entities = [ent for ent in true_named_entities if ent.e_type in tags]\n",
        "    pred_named_entities = [ent for ent in pred_named_entities if ent.e_type in tags]\n",
        "\n",
        "    # go through each predicted named-entity\n",
        "\n",
        "    for pred in pred_named_entities:\n",
        "        found_overlap = False\n",
        "\n",
        "        # Check each of the potential scenarios in turn. See\n",
        "        # http://www.davidsbatista.net/blog/2018/05/09/Named_Entity_Evaluation/\n",
        "        # for scenario explanation.\n",
        "\n",
        "        # Scenario I: Exact match between true and pred\n",
        "\n",
        "        if pred in true_named_entities:\n",
        "            true_which_overlapped_with_pred.append(pred)\n",
        "            evaluation['strict']['correct'] += 1\n",
        "            evaluation['ent_type']['correct'] += 1\n",
        "            evaluation['exact']['correct'] += 1\n",
        "            evaluation['partial']['correct'] += 1\n",
        "\n",
        "            # for the agg. by e_type results\n",
        "            evaluation_agg_entities_type[pred.e_type]['strict']['correct'] += 1\n",
        "            evaluation_agg_entities_type[pred.e_type]['ent_type']['correct'] += 1\n",
        "            evaluation_agg_entities_type[pred.e_type]['exact']['correct'] += 1\n",
        "            evaluation_agg_entities_type[pred.e_type]['partial']['correct'] += 1\n",
        "\n",
        "        else:\n",
        "\n",
        "            # check for overlaps with any of the true entities\n",
        "\n",
        "            for true in true_named_entities:\n",
        "\n",
        "                pred_range = range(pred.start_offset, pred.end_offset)\n",
        "                true_range = range(true.start_offset, true.end_offset)\n",
        "\n",
        "                # Scenario IV: Offsets match, but entity type is wrong\n",
        "\n",
        "                if true.start_offset == pred.start_offset and pred.end_offset == true.end_offset \\\n",
        "                        and true.e_type != pred.e_type:\n",
        "\n",
        "                    # overall results\n",
        "                    evaluation['strict']['incorrect'] += 1\n",
        "                    evaluation['ent_type']['incorrect'] += 1\n",
        "                    evaluation['partial']['correct'] += 1\n",
        "                    evaluation['exact']['correct'] += 1\n",
        "\n",
        "                    # aggregated by entity type results\n",
        "                    evaluation_agg_entities_type[true.e_type]['strict']['incorrect'] += 1\n",
        "                    evaluation_agg_entities_type[true.e_type]['ent_type']['incorrect'] += 1\n",
        "                    evaluation_agg_entities_type[true.e_type]['partial']['correct'] += 1\n",
        "                    evaluation_agg_entities_type[true.e_type]['exact']['correct'] += 1\n",
        "\n",
        "                    true_which_overlapped_with_pred.append(true)\n",
        "                    found_overlap = True\n",
        "\n",
        "                    break\n",
        "\n",
        "                # check for an overlap i.e. not exact boundary match, with true entities\n",
        "\n",
        "                elif find_overlap(true_range, pred_range):\n",
        "\n",
        "                    true_which_overlapped_with_pred.append(true)\n",
        "\n",
        "                    # Scenario V: There is an overlap (but offsets do not match\n",
        "                    # exactly), and the entity type is the same.\n",
        "                    # 2.1 overlaps with the same entity type\n",
        "\n",
        "                    if pred.e_type == true.e_type:\n",
        "\n",
        "                        # overall results\n",
        "                        evaluation['strict']['incorrect'] += 1\n",
        "                        evaluation['ent_type']['correct'] += 1\n",
        "                        evaluation['partial']['partial'] += 1\n",
        "                        evaluation['exact']['incorrect'] += 1\n",
        "\n",
        "                        # aggregated by entity type results\n",
        "                        evaluation_agg_entities_type[true.e_type]['strict']['incorrect'] += 1\n",
        "                        evaluation_agg_entities_type[true.e_type]['ent_type']['correct'] += 1\n",
        "                        evaluation_agg_entities_type[true.e_type]['partial']['partial'] += 1\n",
        "                        evaluation_agg_entities_type[true.e_type]['exact']['incorrect'] += 1\n",
        "\n",
        "                        found_overlap = True\n",
        "\n",
        "                        break\n",
        "\n",
        "                    # Scenario VI: Entities overlap, but the entity type is\n",
        "                    # different.\n",
        "\n",
        "                    else:\n",
        "                        # overall results\n",
        "                        evaluation['strict']['incorrect'] += 1\n",
        "                        evaluation['ent_type']['incorrect'] += 1\n",
        "                        evaluation['partial']['partial'] += 1\n",
        "                        evaluation['exact']['incorrect'] += 1\n",
        "\n",
        "                        # aggregated by entity type results\n",
        "                        # Results against the true entity\n",
        "                        # print(pred)\n",
        "                        # print(true)\n",
        "                        evaluation_agg_entities_type[true.e_type]['strict']['incorrect'] += 1\n",
        "                        evaluation_agg_entities_type[true.e_type]['partial']['partial'] += 1\n",
        "                        evaluation_agg_entities_type[true.e_type]['ent_type']['incorrect'] += 1\n",
        "                        evaluation_agg_entities_type[true.e_type]['exact']['incorrect'] += 1\n",
        "\n",
        "                        # Results against the predicted entity\n",
        "\n",
        "                        # evaluation_agg_entities_type[pred.e_type]['strict']['spurious'] += 1\n",
        "\n",
        "                        found_overlap = True\n",
        "\n",
        "                        break\n",
        "\n",
        "            # Scenario II: Entities are spurious (i.e., over-generated).\n",
        "\n",
        "            if not found_overlap:\n",
        "\n",
        "                # Overall results\n",
        "\n",
        "                evaluation['strict']['spurious'] += 1\n",
        "                evaluation['ent_type']['spurious'] += 1\n",
        "                evaluation['partial']['spurious'] += 1\n",
        "                evaluation['exact']['spurious'] += 1\n",
        "                # print(pred)\n",
        "                # Aggregated by entity type results\n",
        "                # print('Pred : ' ,pred)\n",
        "                evaluation_agg_entities_type[pred.e_type]['strict']['spurious'] += 1\n",
        "                evaluation_agg_entities_type[pred.e_type]['partial']['spurious'] += 1\n",
        "                evaluation_agg_entities_type[pred.e_type]['ent_type']['spurious'] += 1\n",
        "                evaluation_agg_entities_type[pred.e_type]['exact']['spurious'] += 1\n",
        "\n",
        "                # NOTE: when pred.e_type is not found in tags\n",
        "                # or when it simply does not appear in the test set, then it is\n",
        "                # spurious, but it is not clear where to assign it at the tag\n",
        "                # level. In this case, it is applied to all target_tags\n",
        "                # found in this example. This will mean that the sum of the\n",
        "                # evaluation_agg_entities will not equal evaluation.\n",
        "\n",
        "                # for true in tags:                    \n",
        "                #     print('True : ' ,true)\n",
        "                #     evaluation_agg_entities_type[true]['strict']['spurious'] += 1\n",
        "                #     evaluation_agg_entities_type[true]['ent_type']['spurious'] += 1\n",
        "                #     evaluation_agg_entities_type[true]['partial']['spurious'] += 1\n",
        "                #     evaluation_agg_entities_type[true]['exact']['spurious'] += 1\n",
        "\n",
        "    # Scenario III: Entity was missed entirely.\n",
        "\n",
        "    for true in true_named_entities:\n",
        "        if true in true_which_overlapped_with_pred:\n",
        "            continue\n",
        "        else:\n",
        "            # overall results\n",
        "            evaluation['strict']['missed'] += 1\n",
        "            evaluation['ent_type']['missed'] += 1\n",
        "            evaluation['partial']['missed'] += 1\n",
        "            evaluation['exact']['missed'] += 1\n",
        "\n",
        "            # for the agg. by e_type\n",
        "            evaluation_agg_entities_type[true.e_type]['strict']['missed'] += 1\n",
        "            evaluation_agg_entities_type[true.e_type]['ent_type']['missed'] += 1\n",
        "            evaluation_agg_entities_type[true.e_type]['partial']['missed'] += 1\n",
        "            evaluation_agg_entities_type[true.e_type]['exact']['missed'] += 1\n",
        "\n",
        "    # Compute 'possible', 'actual' according to SemEval-2013 Task 9.1 on the\n",
        "    # overall results, and use these to calculate precision and recall.\n",
        "\n",
        "    for eval_type in evaluation:\n",
        "        evaluation[eval_type] = compute_actual_possible(evaluation[eval_type])\n",
        "\n",
        "    # Compute 'possible', 'actual', and precision and recall on entity level\n",
        "    # results. Start by cycling through the accumulated results.\n",
        "\n",
        "    for entity_type, entity_level in evaluation_agg_entities_type.items():\n",
        "\n",
        "        # Cycle through the evaluation types for each dict containing entity\n",
        "        # level results.\n",
        "\n",
        "        for eval_type in entity_level:\n",
        "\n",
        "            evaluation_agg_entities_type[entity_type][eval_type] = compute_actual_possible(\n",
        "                entity_level[eval_type]\n",
        "            )\n",
        "\n",
        "    return evaluation, evaluation_agg_entities_type\n",
        "\n",
        "\n",
        "def find_overlap(true_range, pred_range):\n",
        "    \"\"\"Find the overlap between two ranges\n",
        "\n",
        "    Find the overlap between two ranges. Return the overlapping values if\n",
        "    present, else return an empty set().\n",
        "\n",
        "    Examples:\n",
        "\n",
        "    >>> find_overlap((1, 2), (2, 3))\n",
        "    2\n",
        "    >>> find_overlap((1, 2), (3, 4))\n",
        "    set()\n",
        "    \"\"\"\n",
        "\n",
        "    true_set = set(true_range)\n",
        "    pred_set = set(pred_range)\n",
        "\n",
        "    overlaps = true_set.intersection(pred_set)\n",
        "\n",
        "    return overlaps\n",
        "\n",
        "\n",
        "def compute_actual_possible(results):\n",
        "    \"\"\"\n",
        "    Takes a result dict that has been output by compute metrics.\n",
        "    Returns the results dict with actual, possible populated.\n",
        "\n",
        "    When the results dicts is from partial or ent_type metrics, then\n",
        "    partial_or_type=True to ensure the right calculation is used for\n",
        "    calculating precision and recall.\n",
        "    \"\"\"\n",
        "\n",
        "    correct = results['correct']\n",
        "    incorrect = results['incorrect']\n",
        "    partial = results['partial']\n",
        "    missed = results['missed']\n",
        "    spurious = results['spurious']\n",
        "\n",
        "    # Possible: number annotations in the gold-standard which contribute to the\n",
        "    # final score\n",
        "\n",
        "    possible = correct + incorrect + partial + missed\n",
        "\n",
        "    # Actual: number of annotations produced by the NER system\n",
        "\n",
        "    actual = correct + incorrect + spurious\n",
        "\n",
        "    results[\"actual\"] = actual\n",
        "    results[\"possible\"] = possible\n",
        "\n",
        "    return results\n",
        "\n",
        "\n",
        "def compute_precision_recall(results, partial_or_type=False):\n",
        "    \"\"\"\n",
        "    Takes a result dict that has been output by compute metrics.\n",
        "    Returns the results dict with precison and recall populated.\n",
        "\n",
        "    When the results dicts is from partial or ent_type metrics, then\n",
        "    partial_or_type=True to ensure the right calculation is used for\n",
        "    calculating precision and recall.\n",
        "    \"\"\"\n",
        "\n",
        "    actual = results[\"actual\"]\n",
        "    possible = results[\"possible\"]\n",
        "    partial = results['partial']\n",
        "    correct = results['correct']\n",
        "\n",
        "    if partial_or_type:\n",
        "        precision = (correct + 0.5 * partial) / actual if actual > 0 else 0\n",
        "        recall = (correct + 0.5 * partial) / possible if possible > 0 else 0\n",
        "\n",
        "    else:\n",
        "        precision = correct / actual if actual > 0 else 0\n",
        "        recall = correct / possible if possible > 0 else 0\n",
        "\n",
        "    results[\"precision\"] = precision\n",
        "    results[\"recall\"] = recall\n",
        "\n",
        "    return results\n",
        "\n",
        "\n",
        "def compute_precision_recall_wrapper(results):\n",
        "    \"\"\"\n",
        "    Wraps the compute_precision_recall function and runs on a dict of results\n",
        "    \"\"\"\n",
        "\n",
        "    results_a = {key: compute_precision_recall(value, True) for key, value in results.items() if\n",
        "                 key in ['partial', 'ent_type']}\n",
        "    results_b = {key: compute_precision_recall(value) for key, value in results.items() if\n",
        "                 key in ['strict', 'exact']}\n",
        "\n",
        "    results = {**results_a, **results_b}\n",
        "\n",
        "    return results\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hXB-4VD-zcl"
      },
      "source": [
        "def map_index2label(text, labels):\n",
        "  index2label = {}\n",
        "  start_index = 0\n",
        "  for i, word in enumerate(text):\n",
        "    end_index = start_index + len(word)\n",
        "    index2label[(start_index, end_index)] = labels[i]\n",
        "    start_index = end_index + 1\n",
        "  return index2label"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S0H4XRRN-e4N"
      },
      "source": [
        "def align_prediction(texts, labels_list, model, tag_map):\n",
        "  y_true = []\n",
        "  y_pred = []\n",
        "  index = 0\n",
        "  for text, labels in zip(texts, labels_list):\n",
        "    index2label_true = map_index2label(text, labels)\n",
        "    y_true += [value for key, value in index2label_true.items()]\n",
        "    model_result = model(\" \".join(text))\n",
        "    index2label_pred = {}\n",
        "    for key, value in index2label_true.items():\n",
        "      temp = []\n",
        "      for item in model_result:\n",
        "        if item['start'] >= key[0] and item['end'] <= key[1]:\n",
        "          temp.append(item['entity'])\n",
        "      index2label_pred[(key[0], key[1])] = temp[0]\n",
        "    y_pred += [ tag_map[value] for key, value in index2label_pred.items()]\n",
        "    index += 1\n",
        "    if index%10 == 0:\n",
        "      print(index)\n",
        "    if index==500:\n",
        "      break\n",
        "  return [y_true], [y_pred]"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VVIf6ks69sfb"
      },
      "source": [
        "def benchmark(y_true, y_pred, defined_labels_to_evaluate):\n",
        "  evaluator = Evaluator(y_true, y_pred, tags=defined_labels_to_evaluate)\n",
        "  return evaluator.evaluate()"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HnUOh7OPA1oI"
      },
      "source": [
        "### Albert"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FdjgOPqlAs5K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "740a990cb09140e79886b4afb305dde6",
            "8867af95026e4c048a3503a565477eeb",
            "6f5bfd2fb2ba417eb0429fb48b735112",
            "f2c883cec9544bd7a35adcc2f58e4764",
            "076f7ce5b7b94d4fbbafce1008cd83dd",
            "d5bcdf38330c48a088fb924f321eaf29",
            "84b6933d7fde4b1db19f7b773cce2a36",
            "75fb2c64430a42138907763af018127e",
            "062ad4150ca54156b4dcf90e0991aef9",
            "d9149f0b672448589fd5ab47d1db032b",
            "928676766e2443b6b7c8dabc1c8957df",
            "ee4143debdad40779adcc0ed6c71a0e4",
            "26a0b058956b49c18f36e503c8c89906",
            "221d4d144fea44db9f02e935389c9fb6",
            "90f09e872db44524ad420577e29cbf4b",
            "6b735b6a55e74c61a27084e023cb514a",
            "38adfcd9dcc44f23a4ee98a0a2405ff9",
            "b4f6a44340b34560a773129706a89c1e",
            "3ee1c7ad1021482c9912d66ef9ad6637",
            "b21fcf1b2c9d46b58a1eb317a3fd152f",
            "876286bf35584c3db9820fd0690a000c",
            "c0e51f50612d4fcf94cb898d2c4116fe",
            "260cd720471d4ba59382319e990455d2",
            "d84304e26ae8483eb1ddf6db43529631",
            "223bf9f3a2a248f2a95f1a4d5b281a82",
            "c2190fc617774933989cda2d31a41eab",
            "843f0f9ca535498c86e20f3c476ac4d6",
            "7fb0f5757f7c4b5194b44d33a536808a",
            "4f3a92da5e58450cbe02b4c1ecef3fe3",
            "b80eb194e2364533885857b5971397ca",
            "6b16d40e967a47dc9eb8b0a64c537605",
            "835ee161441841f69458e7ae27ec1b1e",
            "3e5fc997facf41ec9301a0272891c0aa",
            "b0dd8362bb39417994dccdf0bf4209b0",
            "3c31655782794204a5f948eae8562195",
            "f02a80a2737e4fc6b409aa35c658dec5",
            "a288b3cfbab345d9949076215427d92c",
            "484dd7dbb340436696813bebb6fa7c88",
            "469c0fe631aa475da2350272c7403254",
            "27ebb972859f4f59bc28d1f2803501d3",
            "8315bd581e0d49f5ba193ab8f9fb2dbe",
            "4afdc58abb454b408524e482cc566446",
            "2d56ec1e4a40401dab715c15cceabecb",
            "0fe3844cbcba40c0a36d76997299f29e",
            "95277d62192d48ffafb0f713cc739677",
            "cf857c52a61947cea95f4a2940f6d104",
            "8145895b2d0f4a479f839f65ea01fa15",
            "1a35dc523ee84f2493d69c40cc758b7b"
          ]
        },
        "outputId": "bc6dd45f-7b79-40ba-f795-85d1f122b88c"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"HooshvareLab/albert-fa-zwnj-base-v2-ner\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"HooshvareLab/albert-fa-zwnj-base-v2-ner\")\n",
        "model.eval()\n",
        "albert_ner = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:53 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:53 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:46:53 filelock DEBUG: Attempting to acquire lock 140377837699664 on /root/.cache/huggingface/transformers/fa5b2b8037a29a88c692d6aa8acb666ea97c74b19ae19cf4469412f27dbdcb2c.b8e9dbd19707b0813c96c809830537f4417f29e1e032848a54aa349912414965.lock\n",
            "2021-04-15 11:46:53 filelock INFO: Lock 140377837699664 acquired on /root/.cache/huggingface/transformers/fa5b2b8037a29a88c692d6aa8acb666ea97c74b19ae19cf4469412f27dbdcb2c.b8e9dbd19707b0813c96c809830537f4417f29e1e032848a54aa349912414965.lock\n",
            "2021-04-15 11:46:53 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:54 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/config.json HTTP/1.1\" 200 1572\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "740a990cb09140e79886b4afb305dde6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1572.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:54 filelock DEBUG: Attempting to release lock 140377837699664 on /root/.cache/huggingface/transformers/fa5b2b8037a29a88c692d6aa8acb666ea97c74b19ae19cf4469412f27dbdcb2c.b8e9dbd19707b0813c96c809830537f4417f29e1e032848a54aa349912414965.lock\n",
            "2021-04-15 11:46:54 filelock INFO: Lock 140377837699664 released on /root/.cache/huggingface/transformers/fa5b2b8037a29a88c692d6aa8acb666ea97c74b19ae19cf4469412f27dbdcb2c.b8e9dbd19707b0813c96c809830537f4417f29e1e032848a54aa349912414965.lock\n",
            "2021-04-15 11:46:54 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:54 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/spiece.model HTTP/1.1\" 302 0\n",
            "2021-04-15 11:46:54 filelock DEBUG: Attempting to acquire lock 140377837446096 on /root/.cache/huggingface/transformers/1e61e7998995a2c7b20ae37f17dbd451a93bd0325ee7871fa9992cb75efeff10.d998083122643312c83ef4329b4631782a875c6ee59927666f3a972d118e6c56.lock\n",
            "2021-04-15 11:46:54 filelock INFO: Lock 140377837446096 acquired on /root/.cache/huggingface/transformers/1e61e7998995a2c7b20ae37f17dbd451a93bd0325ee7871fa9992cb75efeff10.d998083122643312c83ef4329b4631782a875c6ee59927666f3a972d118e6c56.lock\n",
            "2021-04-15 11:46:54 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-15 11:46:54 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /HooshvareLab/albert-fa-zwnj-base-v2-ner/903319b1a4a7e58e49383764d33897a7f49784510247d68438e4f3bff25b01f1 HTTP/1.1\" 200 857476\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "062ad4150ca54156b4dcf90e0991aef9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=857476.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:54 filelock DEBUG: Attempting to release lock 140377837446096 on /root/.cache/huggingface/transformers/1e61e7998995a2c7b20ae37f17dbd451a93bd0325ee7871fa9992cb75efeff10.d998083122643312c83ef4329b4631782a875c6ee59927666f3a972d118e6c56.lock\n",
            "2021-04-15 11:46:54 filelock INFO: Lock 140377837446096 released on /root/.cache/huggingface/transformers/1e61e7998995a2c7b20ae37f17dbd451a93bd0325ee7871fa9992cb75efeff10.d998083122643312c83ef4329b4631782a875c6ee59927666f3a972d118e6c56.lock\n",
            "2021-04-15 11:46:54 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:55 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/tokenizer.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:46:55 filelock DEBUG: Attempting to acquire lock 140377838575952 on /root/.cache/huggingface/transformers/309e59ec44042dd72731f77ac9f1b3ea7f27e798b4dda075b4e5b574cfc02823.be961fb8b21d67dbb8485194ab3ba872cca87271b562bf02a994d188607ca243.lock\n",
            "2021-04-15 11:46:55 filelock INFO: Lock 140377838575952 acquired on /root/.cache/huggingface/transformers/309e59ec44042dd72731f77ac9f1b3ea7f27e798b4dda075b4e5b574cfc02823.be961fb8b21d67dbb8485194ab3ba872cca87271b562bf02a994d188607ca243.lock\n",
            "2021-04-15 11:46:55 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:55 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/tokenizer.json HTTP/1.1\" 200 3229336\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "38adfcd9dcc44f23a4ee98a0a2405ff9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=3229336.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:56 filelock DEBUG: Attempting to release lock 140377838575952 on /root/.cache/huggingface/transformers/309e59ec44042dd72731f77ac9f1b3ea7f27e798b4dda075b4e5b574cfc02823.be961fb8b21d67dbb8485194ab3ba872cca87271b562bf02a994d188607ca243.lock\n",
            "2021-04-15 11:46:56 filelock INFO: Lock 140377838575952 released on /root/.cache/huggingface/transformers/309e59ec44042dd72731f77ac9f1b3ea7f27e798b4dda075b4e5b574cfc02823.be961fb8b21d67dbb8485194ab3ba872cca87271b562bf02a994d188607ca243.lock\n",
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:46:56 filelock DEBUG: Attempting to acquire lock 140377838576400 on /root/.cache/huggingface/transformers/ca8b9cdacd210800376d99bc8fa4ff0ad3ec558f1546ea0edc09066878079e8d.923ec3af02797bf352e5d19ea7c70e2afaa87ef38bcdc8088cbe93a9bf2ba9ab.lock\n",
            "2021-04-15 11:46:56 filelock INFO: Lock 140377838576400 acquired on /root/.cache/huggingface/transformers/ca8b9cdacd210800376d99bc8fa4ff0ad3ec558f1546ea0edc09066878079e8d.923ec3af02797bf352e5d19ea7c70e2afaa87ef38bcdc8088cbe93a9bf2ba9ab.lock\n",
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/special_tokens_map.json HTTP/1.1\" 200 186\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "223bf9f3a2a248f2a95f1a4d5b281a82",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=186.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:56 filelock DEBUG: Attempting to release lock 140377838576400 on /root/.cache/huggingface/transformers/ca8b9cdacd210800376d99bc8fa4ff0ad3ec558f1546ea0edc09066878079e8d.923ec3af02797bf352e5d19ea7c70e2afaa87ef38bcdc8088cbe93a9bf2ba9ab.lock\n",
            "2021-04-15 11:46:56 filelock INFO: Lock 140377838576400 released on /root/.cache/huggingface/transformers/ca8b9cdacd210800376d99bc8fa4ff0ad3ec558f1546ea0edc09066878079e8d.923ec3af02797bf352e5d19ea7c70e2afaa87ef38bcdc8088cbe93a9bf2ba9ab.lock\n",
            "2021-04-15 11:46:56 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:57 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:46:57 filelock DEBUG: Attempting to acquire lock 140377837380304 on /root/.cache/huggingface/transformers/48a4c7ed555c3de1299c3fc7a2163800468ce034a5d81f9bfe337882c1a88cf4.a89c16d93351e1a6169a82c9eb6324edc64ff127f4ec6a9a72f6462a98aaf017.lock\n",
            "2021-04-15 11:46:57 filelock INFO: Lock 140377837380304 acquired on /root/.cache/huggingface/transformers/48a4c7ed555c3de1299c3fc7a2163800468ce034a5d81f9bfe337882c1a88cf4.a89c16d93351e1a6169a82c9eb6324edc64ff127f4ec6a9a72f6462a98aaf017.lock\n",
            "2021-04-15 11:46:57 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:57 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/tokenizer_config.json HTTP/1.1\" 200 499\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e5fc997facf41ec9301a0272891c0aa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=499.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:57 filelock DEBUG: Attempting to release lock 140377837380304 on /root/.cache/huggingface/transformers/48a4c7ed555c3de1299c3fc7a2163800468ce034a5d81f9bfe337882c1a88cf4.a89c16d93351e1a6169a82c9eb6324edc64ff127f4ec6a9a72f6462a98aaf017.lock\n",
            "2021-04-15 11:46:57 filelock INFO: Lock 140377837380304 released on /root/.cache/huggingface/transformers/48a4c7ed555c3de1299c3fc7a2163800468ce034a5d81f9bfe337882c1a88cf4.a89c16d93351e1a6169a82c9eb6324edc64ff127f4ec6a9a72f6462a98aaf017.lock\n",
            "2021-04-15 11:46:57 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:57 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:46:57 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:46:58 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/albert-fa-zwnj-base-v2-ner/resolve/main/pytorch_model.bin HTTP/1.1\" 302 0\n",
            "2021-04-15 11:46:58 filelock DEBUG: Attempting to acquire lock 140377837382800 on /root/.cache/huggingface/transformers/7b0f0e27645e482092088f175e8c18e4e349fe940811383871e74b60a5ac6fa2.15b683ef18b8b9444c37732797d895b000f4badc046ed32fa916f88decc7ed1d.lock\n",
            "2021-04-15 11:46:58 filelock INFO: Lock 140377837382800 acquired on /root/.cache/huggingface/transformers/7b0f0e27645e482092088f175e8c18e4e349fe940811383871e74b60a5ac6fa2.15b683ef18b8b9444c37732797d895b000f4badc046ed32fa916f88decc7ed1d.lock\n",
            "2021-04-15 11:46:58 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-15 11:46:58 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /HooshvareLab/albert-fa-zwnj-base-v2-ner/0edc50c4a65288408d25530df7a63bfca9efe75fdd94c40641c4e4ea9bef7caf HTTP/1.1\" 200 44451929\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8315bd581e0d49f5ba193ab8f9fb2dbe",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=44451929.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:46:59 filelock DEBUG: Attempting to release lock 140377837382800 on /root/.cache/huggingface/transformers/7b0f0e27645e482092088f175e8c18e4e349fe940811383871e74b60a5ac6fa2.15b683ef18b8b9444c37732797d895b000f4badc046ed32fa916f88decc7ed1d.lock\n",
            "2021-04-15 11:46:59 filelock INFO: Lock 140377837382800 released on /root/.cache/huggingface/transformers/7b0f0e27645e482092088f175e8c18e4e349fe940811383871e74b60a5ac6fa2.15b683ef18b8b9444c37732797d895b000f4badc046ed32fa916f88decc7ed1d.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5G7sgdpJwHB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "36bcb3c4-2f29-4192-fb7b-399b1d9f1da4"
      },
      "source": [
        "  tag_map = {\n",
        "    \"B-DAT\": 'O',\n",
        "    \"B-EVE\": \"B-EVE\",\n",
        "    \"B-FAC\": \"B-ORG\",\n",
        "    \"B-LOC\": \"B-LOC\",\n",
        "    \"B-MON\": \"O\",\n",
        "    \"B-ORG\": \"B-ORG\",\n",
        "    \"B-PER\": \"B-PER\",\n",
        "    \"B-PRO\": \"O\",\n",
        "    \"B-TIM\": \"O\",\n",
        "    \"B-PCT\": \"O\",\n",
        "    \"I-DAT\": \"O\",\n",
        "    \"I-EVE\": \"I-EVE\",\n",
        "    \"I-FAC\": \"I-ORG\",\n",
        "    \"I-LOC\": \"I-LOC\",\n",
        "    \"I-MON\": \"O\",\n",
        "    \"I-ORG\": \"I-ORG\",\n",
        "    \"I-PER\": \"I-PER\",\n",
        "    \"I-PRO\": \"O\",\n",
        "    \"I-TIM\": \"O\",\n",
        "    \"I-PCT\": \"O\",\n",
        "    'O': 'O'\n",
        "  }\n",
        "y_true, y_perd = align_prediction(texts, tags, albert_ner, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:12 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 591,\n",
              "   'correct': 391,\n",
              "   'incorrect': 46,\n",
              "   'missed': 173,\n",
              "   'partial': 0,\n",
              "   'possible': 610,\n",
              "   'precision': 0.6615905245346869,\n",
              "   'recall': 0.6409836065573771,\n",
              "   'spurious': 154},\n",
              "  'exact': {'actual': 591,\n",
              "   'correct': 403,\n",
              "   'incorrect': 34,\n",
              "   'missed': 173,\n",
              "   'partial': 0,\n",
              "   'possible': 610,\n",
              "   'precision': 0.6818950930626058,\n",
              "   'recall': 0.660655737704918,\n",
              "   'spurious': 154},\n",
              "  'partial': {'actual': 557,\n",
              "   'correct': 403,\n",
              "   'incorrect': 0,\n",
              "   'missed': 173,\n",
              "   'partial': 34,\n",
              "   'possible': 610,\n",
              "   'precision': 0.7540394973070018,\n",
              "   'recall': 0.6885245901639344,\n",
              "   'spurious': 154},\n",
              "  'strict': {'actual': 591,\n",
              "   'correct': 365,\n",
              "   'incorrect': 72,\n",
              "   'missed': 173,\n",
              "   'partial': 0,\n",
              "   'possible': 610,\n",
              "   'precision': 0.6175972927241963,\n",
              "   'recall': 0.5983606557377049,\n",
              "   'spurious': 154}},\n",
              " {'EVE': {'ent_type': {'actual': 1,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 1,\n",
              "    'correct': 0,\n",
              "    'incorrect': 1,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 1,\n",
              "    'possible': 14,\n",
              "    'precision': 0,\n",
              "    'recall': 0.03571428571428571,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 1,\n",
              "    'correct': 0,\n",
              "    'incorrect': 1,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0}},\n",
              "  'LOC': {'ent_type': {'actual': 219,\n",
              "    'correct': 150,\n",
              "    'incorrect': 21,\n",
              "    'missed': 51,\n",
              "    'partial': 0,\n",
              "    'possible': 222,\n",
              "    'precision': 0.684931506849315,\n",
              "    'recall': 0.6756756756756757,\n",
              "    'spurious': 48},\n",
              "   'exact': {'actual': 219,\n",
              "    'correct': 162,\n",
              "    'incorrect': 9,\n",
              "    'missed': 51,\n",
              "    'partial': 0,\n",
              "    'possible': 222,\n",
              "    'precision': 0.7397260273972602,\n",
              "    'recall': 0.7297297297297297,\n",
              "    'spurious': 48},\n",
              "   'partial': {'actual': 210,\n",
              "    'correct': 162,\n",
              "    'incorrect': 0,\n",
              "    'missed': 51,\n",
              "    'partial': 9,\n",
              "    'possible': 222,\n",
              "    'precision': 0.7928571428571428,\n",
              "    'recall': 0.75,\n",
              "    'spurious': 48},\n",
              "   'strict': {'actual': 219,\n",
              "    'correct': 144,\n",
              "    'incorrect': 27,\n",
              "    'missed': 51,\n",
              "    'partial': 0,\n",
              "    'possible': 222,\n",
              "    'precision': 0.6575342465753424,\n",
              "    'recall': 0.6486486486486487,\n",
              "    'spurious': 48}},\n",
              "  'ORG': {'ent_type': {'actual': 140,\n",
              "    'correct': 72,\n",
              "    'incorrect': 19,\n",
              "    'missed': 39,\n",
              "    'partial': 0,\n",
              "    'possible': 130,\n",
              "    'precision': 0.5142857142857142,\n",
              "    'recall': 0.5538461538461539,\n",
              "    'spurious': 49},\n",
              "   'exact': {'actual': 140,\n",
              "    'correct': 73,\n",
              "    'incorrect': 18,\n",
              "    'missed': 39,\n",
              "    'partial': 0,\n",
              "    'possible': 130,\n",
              "    'precision': 0.5214285714285715,\n",
              "    'recall': 0.5615384615384615,\n",
              "    'spurious': 49},\n",
              "   'partial': {'actual': 122,\n",
              "    'correct': 73,\n",
              "    'incorrect': 0,\n",
              "    'missed': 39,\n",
              "    'partial': 18,\n",
              "    'possible': 130,\n",
              "    'precision': 0.6721311475409836,\n",
              "    'recall': 0.6307692307692307,\n",
              "    'spurious': 49},\n",
              "   'strict': {'actual': 140,\n",
              "    'correct': 59,\n",
              "    'incorrect': 32,\n",
              "    'missed': 39,\n",
              "    'partial': 0,\n",
              "    'possible': 130,\n",
              "    'precision': 0.42142857142857143,\n",
              "    'recall': 0.45384615384615384,\n",
              "    'spurious': 49}},\n",
              "  'PER': {'ent_type': {'actual': 231,\n",
              "    'correct': 168,\n",
              "    'incorrect': 6,\n",
              "    'missed': 70,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.7272727272727273,\n",
              "    'recall': 0.6885245901639344,\n",
              "    'spurious': 57},\n",
              "   'exact': {'actual': 231,\n",
              "    'correct': 168,\n",
              "    'incorrect': 6,\n",
              "    'missed': 70,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.7272727272727273,\n",
              "    'recall': 0.6885245901639344,\n",
              "    'spurious': 57},\n",
              "   'partial': {'actual': 225,\n",
              "    'correct': 168,\n",
              "    'incorrect': 0,\n",
              "    'missed': 70,\n",
              "    'partial': 6,\n",
              "    'possible': 244,\n",
              "    'precision': 0.76,\n",
              "    'recall': 0.7008196721311475,\n",
              "    'spurious': 57},\n",
              "   'strict': {'actual': 231,\n",
              "    'correct': 162,\n",
              "    'incorrect': 12,\n",
              "    'missed': 70,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.7012987012987013,\n",
              "    'recall': 0.6639344262295082,\n",
              "    'spurious': 57}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JxcbjYY2GiDT"
      },
      "source": [
        "### Pars Bert"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CrYU_D36Gj_T",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "731ec5d4ac0e4cfea66258b3d99bec52",
            "81d0d46918e84d5885866849f680d9f5",
            "d06e6a3094f44896ac6c9ebfcbe062aa",
            "2f97f24a5fac487586a542682307765c",
            "d4f18e17aebd4a728781843b1ee36c7d",
            "be18f78fc71f47f2ad2e84d6747ecde6",
            "16626393ccdf4eccbba056650e2d0cbb",
            "d69faeb806694e139a1b204ad2e24cbf",
            "f3cfe533414f433686f91e5322b2d2ca",
            "6436abe5211b461d8efa2eb2028f364c",
            "f5e35d72bbaf45d3b79509b37d75552f",
            "97992c39b9054490ba9478d9d8f6cab9",
            "e51245b565364f4883a3f081609a7e17",
            "234e3e547be441e99cfae1c01f7bcee0",
            "deaef04510d2450287e77c807b3a159e",
            "6a5c9f8b96e74fc5abc91bf1d69dcfa5",
            "b4a4420f538c42538f2e545269f9b6cc",
            "c220ab2004d64c8e84d26703c86d43f2",
            "a3fe177fffd24e989754fb05af17639f",
            "d4b72fb5d798433e8c74c25e6405186a",
            "c165cacfebbf4cf696f7df313ea2b1bb",
            "1653272ac5ce44329bea5b5a64bc1407",
            "55cb3114c0e848ee9f22bcee1f134155",
            "6e9d9eb588aa4a5faf106f8fb354d3a0",
            "32f9c9df1efd49c7b306d7cc983195a8",
            "dde6b01fbf504ea785ebc18312769f58",
            "2f115a31a45e4ed5b031682045a812d0",
            "681d83516082405b9cff230a124d223a",
            "b866c6ff72324b97bee39985430e0b5a",
            "b078c027972c43f68c5ec1ca3c1def5e",
            "f164c78f53b744919992bce4bae6edf7",
            "a51a74f4461b43cdb4a9ee6937981ddc",
            "6685c1af24754a9b8afb1dde1389009a",
            "c4ac27e658e047be849e85a5bced10ee",
            "87528c0af63243b19e973150d86b30e3",
            "b9faff595b5e4d20819a7f42704c0067",
            "b33bb1ec47bd443090d77402e463be48",
            "76f703b6be394f6fbaa9e5bc9132f4ae",
            "a728cccb6f3a464b8ed2d5a18ea0f66b",
            "6ffe46e435b349f8a8e7833b7153860f"
          ]
        },
        "outputId": "cabf07ad-6168-4efd-da1a-9ec84faa3fa4"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"HooshvareLab/bert-base-parsbert-ner-uncased\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"HooshvareLab/bert-base-parsbert-ner-uncased\")\n",
        "model.eval()\n",
        "parsbert_ner = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:48:12 filelock DEBUG: Attempting to acquire lock 140377830001744 on /root/.cache/huggingface/transformers/cae79f4815c0cf33be75b6a1aa1d6236ab2646d25c5c26f5083ec2f31a6bc82f.c3f34538a8ec527cdbc5de919e808441d6737abb85fe85a68db484f12dceb83a.lock\n",
            "2021-04-15 11:48:12 filelock INFO: Lock 140377830001744 acquired on /root/.cache/huggingface/transformers/cae79f4815c0cf33be75b6a1aa1d6236ab2646d25c5c26f5083ec2f31a6bc82f.c3f34538a8ec527cdbc5de919e808441d6737abb85fe85a68db484f12dceb83a.lock\n",
            "2021-04-15 11:48:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:13 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/config.json HTTP/1.1\" 200 1357\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "731ec5d4ac0e4cfea66258b3d99bec52",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1357.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:13 filelock DEBUG: Attempting to release lock 140377830001744 on /root/.cache/huggingface/transformers/cae79f4815c0cf33be75b6a1aa1d6236ab2646d25c5c26f5083ec2f31a6bc82f.c3f34538a8ec527cdbc5de919e808441d6737abb85fe85a68db484f12dceb83a.lock\n",
            "2021-04-15 11:48:13 filelock INFO: Lock 140377830001744 released on /root/.cache/huggingface/transformers/cae79f4815c0cf33be75b6a1aa1d6236ab2646d25c5c26f5083ec2f31a6bc82f.c3f34538a8ec527cdbc5de919e808441d6737abb85fe85a68db484f12dceb83a.lock\n",
            "2021-04-15 11:48:13 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:13 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-15 11:48:13 filelock DEBUG: Attempting to acquire lock 140377830232016 on /root/.cache/huggingface/transformers/5df91a905c8036634a1cddbbcdab55e558aec2fd4309a294217d01f403e371f7.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-15 11:48:13 filelock INFO: Lock 140377830232016 acquired on /root/.cache/huggingface/transformers/5df91a905c8036634a1cddbbcdab55e558aec2fd4309a294217d01f403e371f7.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-15 11:48:13 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:13 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 1215509\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f3cfe533414f433686f91e5322b2d2ca",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:14 filelock DEBUG: Attempting to release lock 140377830232016 on /root/.cache/huggingface/transformers/5df91a905c8036634a1cddbbcdab55e558aec2fd4309a294217d01f403e371f7.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-15 11:48:14 filelock INFO: Lock 140377830232016 released on /root/.cache/huggingface/transformers/5df91a905c8036634a1cddbbcdab55e558aec2fd4309a294217d01f403e371f7.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-15 11:48:14 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:14 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-15 11:48:14 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:14 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-15 11:48:14 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:15 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:48:15 filelock DEBUG: Attempting to acquire lock 140377830230032 on /root/.cache/huggingface/transformers/9c399f248ec41d037b934b424658c33cd7050d5eb47eadfaaebef9859880f728.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-15 11:48:15 filelock INFO: Lock 140377830230032 acquired on /root/.cache/huggingface/transformers/9c399f248ec41d037b934b424658c33cd7050d5eb47eadfaaebef9859880f728.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-15 11:48:15 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:15 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 112\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b4a4420f538c42538f2e545269f9b6cc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:15 filelock DEBUG: Attempting to release lock 140377830230032 on /root/.cache/huggingface/transformers/9c399f248ec41d037b934b424658c33cd7050d5eb47eadfaaebef9859880f728.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-15 11:48:15 filelock INFO: Lock 140377830230032 released on /root/.cache/huggingface/transformers/9c399f248ec41d037b934b424658c33cd7050d5eb47eadfaaebef9859880f728.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-15 11:48:15 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:15 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:48:15 filelock DEBUG: Attempting to acquire lock 140377837293008 on /root/.cache/huggingface/transformers/d9c378c1d878d4339ca3cfb8759681d66bb8a5b79d66e9905733939c2e47416e.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-15 11:48:15 filelock INFO: Lock 140377837293008 acquired on /root/.cache/huggingface/transformers/d9c378c1d878d4339ca3cfb8759681d66bb8a5b79d66e9905733939c2e47416e.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-15 11:48:15 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:16 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "32f9c9df1efd49c7b306d7cc983195a8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:16 filelock DEBUG: Attempting to release lock 140377837293008 on /root/.cache/huggingface/transformers/d9c378c1d878d4339ca3cfb8759681d66bb8a5b79d66e9905733939c2e47416e.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-15 11:48:16 filelock INFO: Lock 140377837293008 released on /root/.cache/huggingface/transformers/d9c378c1d878d4339ca3cfb8759681d66bb8a5b79d66e9905733939c2e47416e.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:16 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:16 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:48:16 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:48:17 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-ner-uncased/resolve/main/pytorch_model.bin HTTP/1.1\" 302 0\n",
            "2021-04-15 11:48:17 filelock DEBUG: Attempting to acquire lock 140377837156816 on /root/.cache/huggingface/transformers/74434d7ca32de74f5c4e1ef13ddcee2b2fe7b4a2861e8141267ef695011b0df7.aee1184fa3234eec730b51786e2e50883bb3b2e1312308fcb0999d13cb2ba8c5.lock\n",
            "2021-04-15 11:48:17 filelock INFO: Lock 140377837156816 acquired on /root/.cache/huggingface/transformers/74434d7ca32de74f5c4e1ef13ddcee2b2fe7b4a2861e8141267ef695011b0df7.aee1184fa3234eec730b51786e2e50883bb3b2e1312308fcb0999d13cb2ba8c5.lock\n",
            "2021-04-15 11:48:17 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-15 11:48:17 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-ner-uncased/c82171425cd63fef57a706c218f7c01d699ce9371ece9555a1907ddb657cb5ef HTTP/1.1\" 200 651478185\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6685c1af24754a9b8afb1dde1389009a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=651478185.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:48:31 filelock DEBUG: Attempting to release lock 140377837156816 on /root/.cache/huggingface/transformers/74434d7ca32de74f5c4e1ef13ddcee2b2fe7b4a2861e8141267ef695011b0df7.aee1184fa3234eec730b51786e2e50883bb3b2e1312308fcb0999d13cb2ba8c5.lock\n",
            "2021-04-15 11:48:31 filelock INFO: Lock 140377837156816 released on /root/.cache/huggingface/transformers/74434d7ca32de74f5c4e1ef13ddcee2b2fe7b4a2861e8141267ef695011b0df7.aee1184fa3234eec730b51786e2e50883bb3b2e1312308fcb0999d13cb2ba8c5.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-tFCcvzG5a9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a42b9723-2ef7-41b6-ea84-22cf4bdb6f47"
      },
      "source": [
        "  tag_map = {\n",
        "    \"B-date\": 'O',\n",
        "    \"B-event\": \"B-EVE\",\n",
        "    \"B-facility\": \"B-ORG\",\n",
        "    \"B-location\": \"B-LOC\",\n",
        "    \"B-money\": \"O\",\n",
        "    \"B-organization\": \"B-ORG\",\n",
        "    \"B-person\": \"B-PER\",\n",
        "    \"B-product\": \"O\",\n",
        "    \"B-time\": \"O\",\n",
        "    \"B-percent\": \"O\",\n",
        "    \"I-date\": \"O\",\n",
        "    \"I-event\": \"I-EVE\",\n",
        "    \"I-facility\": \"I-ORG\",\n",
        "    \"I-location\": \"I-LOC\",\n",
        "    \"I-money\": \"O\",\n",
        "    \"I-organization\": \"I-ORG\",\n",
        "    \"I-person\": \"I-PER\",\n",
        "    \"I-product\": \"O\",\n",
        "    \"I-time\": \"O\",\n",
        "    \"I-percent\": \"O\",\n",
        "    'O': 'O'\n",
        "  }\n",
        "y_true, y_perd = align_prediction(texts, tags, parsbert_ner, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:30 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 598,\n",
              "   'correct': 434,\n",
              "   'incorrect': 37,\n",
              "   'missed': 137,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.725752508361204,\n",
              "   'recall': 0.7138157894736842,\n",
              "   'spurious': 127},\n",
              "  'exact': {'actual': 598,\n",
              "   'correct': 452,\n",
              "   'incorrect': 19,\n",
              "   'missed': 137,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7558528428093646,\n",
              "   'recall': 0.743421052631579,\n",
              "   'spurious': 127},\n",
              "  'partial': {'actual': 579,\n",
              "   'correct': 452,\n",
              "   'incorrect': 0,\n",
              "   'missed': 137,\n",
              "   'partial': 19,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7970639032815199,\n",
              "   'recall': 0.759046052631579,\n",
              "   'spurious': 127},\n",
              "  'strict': {'actual': 598,\n",
              "   'correct': 419,\n",
              "   'incorrect': 52,\n",
              "   'missed': 137,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7006688963210702,\n",
              "   'recall': 0.6891447368421053,\n",
              "   'spurious': 127}},\n",
              " {'EVE': {'ent_type': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1}},\n",
              "  'LOC': {'ent_type': {'actual': 209,\n",
              "    'correct': 149,\n",
              "    'incorrect': 20,\n",
              "    'missed': 52,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7129186602870813,\n",
              "    'recall': 0.6742081447963801,\n",
              "    'spurious': 40},\n",
              "   'exact': {'actual': 209,\n",
              "    'correct': 165,\n",
              "    'incorrect': 4,\n",
              "    'missed': 52,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7894736842105263,\n",
              "    'recall': 0.746606334841629,\n",
              "    'spurious': 40},\n",
              "   'partial': {'actual': 205,\n",
              "    'correct': 165,\n",
              "    'incorrect': 0,\n",
              "    'missed': 52,\n",
              "    'partial': 4,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8146341463414634,\n",
              "    'recall': 0.755656108597285,\n",
              "    'spurious': 40},\n",
              "   'strict': {'actual': 209,\n",
              "    'correct': 147,\n",
              "    'incorrect': 22,\n",
              "    'missed': 52,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7033492822966507,\n",
              "    'recall': 0.665158371040724,\n",
              "    'spurious': 40}},\n",
              "  'ORG': {'ent_type': {'actual': 143,\n",
              "    'correct': 83,\n",
              "    'incorrect': 11,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5804195804195804,\n",
              "    'recall': 0.6434108527131783,\n",
              "    'spurious': 49},\n",
              "   'exact': {'actual': 143,\n",
              "    'correct': 85,\n",
              "    'incorrect': 9,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5944055944055944,\n",
              "    'recall': 0.6589147286821705,\n",
              "    'spurious': 49},\n",
              "   'partial': {'actual': 134,\n",
              "    'correct': 85,\n",
              "    'incorrect': 0,\n",
              "    'missed': 35,\n",
              "    'partial': 9,\n",
              "    'possible': 129,\n",
              "    'precision': 0.667910447761194,\n",
              "    'recall': 0.6937984496124031,\n",
              "    'spurious': 49},\n",
              "   'strict': {'actual': 143,\n",
              "    'correct': 75,\n",
              "    'incorrect': 19,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5244755244755245,\n",
              "    'recall': 0.5813953488372093,\n",
              "    'spurious': 49}},\n",
              "  'PER': {'ent_type': {'actual': 244,\n",
              "    'correct': 201,\n",
              "    'incorrect': 6,\n",
              "    'missed': 37,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8237704918032787,\n",
              "    'recall': 0.8237704918032787,\n",
              "    'spurious': 37},\n",
              "   'exact': {'actual': 244,\n",
              "    'correct': 201,\n",
              "    'incorrect': 6,\n",
              "    'missed': 37,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8237704918032787,\n",
              "    'recall': 0.8237704918032787,\n",
              "    'spurious': 37},\n",
              "   'partial': {'actual': 238,\n",
              "    'correct': 201,\n",
              "    'incorrect': 0,\n",
              "    'missed': 37,\n",
              "    'partial': 6,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8571428571428571,\n",
              "    'recall': 0.8360655737704918,\n",
              "    'spurious': 37},\n",
              "   'strict': {'actual': 244,\n",
              "    'correct': 196,\n",
              "    'incorrect': 11,\n",
              "    'missed': 37,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8032786885245902,\n",
              "    'recall': 0.8032786885245902,\n",
              "    'spurious': 37}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QINue685-ZQP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2a78facc-ceab-49d4-c916-fc349c94c011"
      },
      "source": [
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:30 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 598,\n",
              "   'correct': 434,\n",
              "   'incorrect': 37,\n",
              "   'missed': 137,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.725752508361204,\n",
              "   'recall': 0.7138157894736842,\n",
              "   'spurious': 127},\n",
              "  'exact': {'actual': 598,\n",
              "   'correct': 452,\n",
              "   'incorrect': 19,\n",
              "   'missed': 137,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7558528428093646,\n",
              "   'recall': 0.743421052631579,\n",
              "   'spurious': 127},\n",
              "  'partial': {'actual': 579,\n",
              "   'correct': 452,\n",
              "   'incorrect': 0,\n",
              "   'missed': 137,\n",
              "   'partial': 19,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7970639032815199,\n",
              "   'recall': 0.759046052631579,\n",
              "   'spurious': 127},\n",
              "  'strict': {'actual': 598,\n",
              "   'correct': 419,\n",
              "   'incorrect': 52,\n",
              "   'missed': 137,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7006688963210702,\n",
              "   'recall': 0.6891447368421053,\n",
              "   'spurious': 127}},\n",
              " {'EVE': {'ent_type': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 2,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07142857142857142,\n",
              "    'spurious': 1}},\n",
              "  'LOC': {'ent_type': {'actual': 209,\n",
              "    'correct': 149,\n",
              "    'incorrect': 20,\n",
              "    'missed': 52,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7129186602870813,\n",
              "    'recall': 0.6742081447963801,\n",
              "    'spurious': 40},\n",
              "   'exact': {'actual': 209,\n",
              "    'correct': 165,\n",
              "    'incorrect': 4,\n",
              "    'missed': 52,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7894736842105263,\n",
              "    'recall': 0.746606334841629,\n",
              "    'spurious': 40},\n",
              "   'partial': {'actual': 205,\n",
              "    'correct': 165,\n",
              "    'incorrect': 0,\n",
              "    'missed': 52,\n",
              "    'partial': 4,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8146341463414634,\n",
              "    'recall': 0.755656108597285,\n",
              "    'spurious': 40},\n",
              "   'strict': {'actual': 209,\n",
              "    'correct': 147,\n",
              "    'incorrect': 22,\n",
              "    'missed': 52,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7033492822966507,\n",
              "    'recall': 0.665158371040724,\n",
              "    'spurious': 40}},\n",
              "  'ORG': {'ent_type': {'actual': 143,\n",
              "    'correct': 83,\n",
              "    'incorrect': 11,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5804195804195804,\n",
              "    'recall': 0.6434108527131783,\n",
              "    'spurious': 49},\n",
              "   'exact': {'actual': 143,\n",
              "    'correct': 85,\n",
              "    'incorrect': 9,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5944055944055944,\n",
              "    'recall': 0.6589147286821705,\n",
              "    'spurious': 49},\n",
              "   'partial': {'actual': 134,\n",
              "    'correct': 85,\n",
              "    'incorrect': 0,\n",
              "    'missed': 35,\n",
              "    'partial': 9,\n",
              "    'possible': 129,\n",
              "    'precision': 0.667910447761194,\n",
              "    'recall': 0.6937984496124031,\n",
              "    'spurious': 49},\n",
              "   'strict': {'actual': 143,\n",
              "    'correct': 75,\n",
              "    'incorrect': 19,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5244755244755245,\n",
              "    'recall': 0.5813953488372093,\n",
              "    'spurious': 49}},\n",
              "  'PER': {'ent_type': {'actual': 244,\n",
              "    'correct': 201,\n",
              "    'incorrect': 6,\n",
              "    'missed': 37,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8237704918032787,\n",
              "    'recall': 0.8237704918032787,\n",
              "    'spurious': 37},\n",
              "   'exact': {'actual': 244,\n",
              "    'correct': 201,\n",
              "    'incorrect': 6,\n",
              "    'missed': 37,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8237704918032787,\n",
              "    'recall': 0.8237704918032787,\n",
              "    'spurious': 37},\n",
              "   'partial': {'actual': 238,\n",
              "    'correct': 201,\n",
              "    'incorrect': 0,\n",
              "    'missed': 37,\n",
              "    'partial': 6,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8571428571428571,\n",
              "    'recall': 0.8360655737704918,\n",
              "    'spurious': 37},\n",
              "   'strict': {'actual': 244,\n",
              "    'correct': 196,\n",
              "    'incorrect': 11,\n",
              "    'missed': 37,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8032786885245902,\n",
              "    'recall': 0.8032786885245902,\n",
              "    'spurious': 37}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pC7ixhiXJqKf"
      },
      "source": [
        "### XLMR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Equ58OwJrhm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "7301d5b2784248dfae506426a58863b6",
            "b4ef81263c8c4472be8521ecfaae4dcf",
            "ae6d12be652749d594cb4148618c90f4",
            "ebdd97ba670d45d8acf68e0c552625fa",
            "22cbc589b23e43a08cb8a70f64bec7bf",
            "14713245725f4090a2cde4ca592366a3",
            "7c21c204c0f8483bb218e374da2715e6",
            "ac67fc675e284cba94d6cf0b11b686b1",
            "fc957ec21b4f4cea81dba07c8c204eef",
            "6ea013abe9224f73bd3c61a07f195006",
            "bb4536cbbf084e6d91fab5f347d84f6a",
            "55c90f639f004efbad7d796694bacc31",
            "9670dc9d89af430d848bb0464cdc55a2",
            "fcefda86bdd64c9a9843bc9a1ce97df2",
            "852b7646a53644a3874626a6f8a18d7d",
            "baa3c022504b4fc1a27a91b849597df0",
            "b46e6e326dc442f49f534be4fd636461",
            "6612f18c66af42378c4f8e7a2d34e239",
            "baf8bfb4787c48ffa29d76a4fcc3ec5d",
            "7873f4a59467498b9e52a70e789a5f10",
            "fa5163b1c23b4c258b7dc15bfbffed85",
            "4bc29d42c8dd4f7284fccc21831328bf",
            "80d9fb8b44034efda2eb9aab27fa6adc",
            "c51dc16fc80d40d888a65e54105fd3a8",
            "c308d2d892d1425bb26ef6a8b4514149",
            "20ffea64882b40f79bcaba662bc61f11",
            "8f1346003b0c41459ac4c0410ee68333",
            "f35472be819545aba946fbbd89e5dbbb",
            "1a4b8ad9a80148f8941486d3aa0714e6",
            "de6e9a4c19db44f19064dcd2eff7537f",
            "cf08174a61b648d28d2c545a30f09b55",
            "4aa938d0d02a4e049b29be0d344e3e5e",
            "9dd8d0d3e8ab4daabba3d2f271763c53",
            "cf26895f2f49427091e88bcfd40019a4",
            "86926ebed1c048d683acb4a260c56c8d",
            "f866b3f8cc4844919b0106af498ab7d4",
            "b8806af543c34003bee7f86f416b4d37",
            "07d34f4737ff44a399904262a629cd2b",
            "1bf6ed64658e42038a9665aa74251783",
            "5d9c5c25f7464fb38754f8463a739ca9"
          ]
        },
        "outputId": "3896db99-caa4-4de7-dbbe-c660113463c5"
      },
      "source": [
        "nlp_ner = pipeline(\n",
        "    \"ner\",\n",
        "    model=\"jplu/tf-xlm-r-ner-40-lang\",\n",
        "    tokenizer=(\n",
        "        'jplu/tf-xlm-r-ner-40-lang',  \n",
        "        {\"use_fast\": True}),\n",
        "    framework=\"tf\",\n",
        "    ignore_labels=[],\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:30 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:30 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:49:30 filelock DEBUG: Attempting to acquire lock 140380418734544 on /root/.cache/huggingface/transformers/592edd8ca19e03c273c0d78107e6114dd71159783068fd8a7055ef64a925bff4.8febe5547bd288d4fdf088fad643ea91ad188f30915e46d3b989a3d06364d222.lock\n",
            "2021-04-15 11:49:30 filelock INFO: Lock 140380418734544 acquired on /root/.cache/huggingface/transformers/592edd8ca19e03c273c0d78107e6114dd71159783068fd8a7055ef64a925bff4.8febe5547bd288d4fdf088fad643ea91ad188f30915e46d3b989a3d06364d222.lock\n",
            "2021-04-15 11:49:30 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:31 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /jplu/tf-xlm-r-ner-40-lang/resolve/main/config.json HTTP/1.1\" 200 699\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7301d5b2784248dfae506426a58863b6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=699.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:31 filelock DEBUG: Attempting to release lock 140380418734544 on /root/.cache/huggingface/transformers/592edd8ca19e03c273c0d78107e6114dd71159783068fd8a7055ef64a925bff4.8febe5547bd288d4fdf088fad643ea91ad188f30915e46d3b989a3d06364d222.lock\n",
            "2021-04-15 11:49:31 filelock INFO: Lock 140380418734544 released on /root/.cache/huggingface/transformers/592edd8ca19e03c273c0d78107e6114dd71159783068fd8a7055ef64a925bff4.8febe5547bd288d4fdf088fad643ea91ad188f30915e46d3b989a3d06364d222.lock\n",
            "2021-04-15 11:49:31 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:31 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/sentencepiece.bpe.model HTTP/1.1\" 200 0\n",
            "2021-04-15 11:49:31 filelock DEBUG: Attempting to acquire lock 140377798113168 on /root/.cache/huggingface/transformers/610d44b2598cdb5337559ed970ff2fc69117c0d36786c2d108aacecc439b4599.00628a9eeb8baf4080d44a0abe9fe8057893de20c7cb6e6423cddbf452f7d4d8.lock\n",
            "2021-04-15 11:49:31 filelock INFO: Lock 140377798113168 acquired on /root/.cache/huggingface/transformers/610d44b2598cdb5337559ed970ff2fc69117c0d36786c2d108aacecc439b4599.00628a9eeb8baf4080d44a0abe9fe8057893de20c7cb6e6423cddbf452f7d4d8.lock\n",
            "2021-04-15 11:49:31 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:31 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /jplu/tf-xlm-r-ner-40-lang/resolve/main/sentencepiece.bpe.model HTTP/1.1\" 200 5069051\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc957ec21b4f4cea81dba07c8c204eef",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=5069051.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:32 filelock DEBUG: Attempting to release lock 140377798113168 on /root/.cache/huggingface/transformers/610d44b2598cdb5337559ed970ff2fc69117c0d36786c2d108aacecc439b4599.00628a9eeb8baf4080d44a0abe9fe8057893de20c7cb6e6423cddbf452f7d4d8.lock\n",
            "2021-04-15 11:49:32 filelock INFO: Lock 140377798113168 released on /root/.cache/huggingface/transformers/610d44b2598cdb5337559ed970ff2fc69117c0d36786c2d108aacecc439b4599.00628a9eeb8baf4080d44a0abe9fe8057893de20c7cb6e6423cddbf452f7d4d8.lock\n",
            "2021-04-15 11:49:32 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:32 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-15 11:49:32 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:49:33 filelock DEBUG: Attempting to acquire lock 140377798113168 on /root/.cache/huggingface/transformers/fb6b2e33c0bb4c25b56431adf3869c18c5a20e80acb8a45672b2020ab6019f03.0dc5b1041f62041ebbd23b1297f2f573769d5c97d8b7c28180ec86b8f6185aa8.lock\n",
            "2021-04-15 11:49:33 filelock INFO: Lock 140377798113168 acquired on /root/.cache/huggingface/transformers/fb6b2e33c0bb4c25b56431adf3869c18c5a20e80acb8a45672b2020ab6019f03.0dc5b1041f62041ebbd23b1297f2f573769d5c97d8b7c28180ec86b8f6185aa8.lock\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /jplu/tf-xlm-r-ner-40-lang/resolve/main/special_tokens_map.json HTTP/1.1\" 200 150\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b46e6e326dc442f49f534be4fd636461",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=150.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:33 filelock DEBUG: Attempting to release lock 140377798113168 on /root/.cache/huggingface/transformers/fb6b2e33c0bb4c25b56431adf3869c18c5a20e80acb8a45672b2020ab6019f03.0dc5b1041f62041ebbd23b1297f2f573769d5c97d8b7c28180ec86b8f6185aa8.lock\n",
            "2021-04-15 11:49:33 filelock INFO: Lock 140377798113168 released on /root/.cache/huggingface/transformers/fb6b2e33c0bb4c25b56431adf3869c18c5a20e80acb8a45672b2020ab6019f03.0dc5b1041f62041ebbd23b1297f2f573769d5c97d8b7c28180ec86b8f6185aa8.lock\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:49:33 filelock DEBUG: Attempting to acquire lock 140377798101968 on /root/.cache/huggingface/transformers/192171f27b852eb9adff1624e0ed6dca9bf1aed82ece5566077669556838808c.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-15 11:49:33 filelock INFO: Lock 140377798101968 acquired on /root/.cache/huggingface/transformers/192171f27b852eb9adff1624e0ed6dca9bf1aed82ece5566077669556838808c.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-15 11:49:33 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:34 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /jplu/tf-xlm-r-ner-40-lang/resolve/main/tokenizer_config.json HTTP/1.1\" 200 2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c308d2d892d1425bb26ef6a8b4514149",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:34 filelock DEBUG: Attempting to release lock 140377798101968 on /root/.cache/huggingface/transformers/192171f27b852eb9adff1624e0ed6dca9bf1aed82ece5566077669556838808c.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-15 11:49:34 filelock INFO: Lock 140377798101968 released on /root/.cache/huggingface/transformers/192171f27b852eb9adff1624e0ed6dca9bf1aed82ece5566077669556838808c.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:49:35 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/modelcard.json HTTP/1.1\" 404 0\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /jplu/tf-xlm-r-ner-40-lang/resolve/main/tf_model.h5 HTTP/1.1\" 302 0\n",
            "2021-04-15 11:49:36 filelock DEBUG: Attempting to acquire lock 140377798113168 on /root/.cache/huggingface/transformers/309b1bef385d3c2d3a7f8e15f32f93a5347508e13edfc0f562999e06a78887e7.52fe00cc1309b8aa154b26b5e0c7b34af351c439d525ff6501d3fe634daf8aef.h5.lock\n",
            "2021-04-15 11:49:36 filelock INFO: Lock 140377798113168 acquired on /root/.cache/huggingface/transformers/309b1bef385d3c2d3a7f8e15f32f93a5347508e13edfc0f562999e06a78887e7.52fe00cc1309b8aa154b26b5e0c7b34af351c439d525ff6501d3fe634daf8aef.h5.lock\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-15 11:49:36 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /jplu/tf-xlm-r-ner-40-lang/8f4a6d5af50d93a36212a159101b4febb5109259cecd74c0292c02b4d40e9b5f HTTP/1.1\" 200 1112459008\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9dd8d0d3e8ab4daabba3d2f271763c53",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1112459008.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:50:01 filelock DEBUG: Attempting to release lock 140377798113168 on /root/.cache/huggingface/transformers/309b1bef385d3c2d3a7f8e15f32f93a5347508e13edfc0f562999e06a78887e7.52fe00cc1309b8aa154b26b5e0c7b34af351c439d525ff6501d3fe634daf8aef.h5.lock\n",
            "2021-04-15 11:50:01 filelock INFO: Lock 140377798113168 released on /root/.cache/huggingface/transformers/309b1bef385d3c2d3a7f8e15f32f93a5347508e13edfc0f562999e06a78887e7.52fe00cc1309b8aa154b26b5e0c7b34af351c439d525ff6501d3fe634daf8aef.h5.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at jplu/tf-xlm-r-ner-40-lang were not used when initializing TFXLMRobertaForTokenClassification: ['dropout_38']\n",
            "- This IS expected if you are initializing TFXLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFXLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFXLMRobertaForTokenClassification were initialized from the model checkpoint at jplu/tf-xlm-r-ner-40-lang.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFXLMRobertaForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zWadF-HwJ5mL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6593c48-4005-48d7-e739-79c5d2417cec"
      },
      "source": [
        "tag_map = {\n",
        "    \"PER\": 'B-PER',\n",
        "    \"LOC\": \"B-LOC\",\n",
        "    \"ORG\": \"B-ORG\",\n",
        "    'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts, tags, nlp_ner, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:50:10 tensorflow WARNING: From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-15 11:51:50 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 1223,\n",
              "   'correct': 247,\n",
              "   'incorrect': 49,\n",
              "   'missed': 298,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.2019623875715454,\n",
              "   'recall': 0.4158249158249158,\n",
              "   'spurious': 927},\n",
              "  'exact': {'actual': 1223,\n",
              "   'correct': 296,\n",
              "   'incorrect': 0,\n",
              "   'missed': 298,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.24202780049059688,\n",
              "   'recall': 0.4983164983164983,\n",
              "   'spurious': 927},\n",
              "  'partial': {'actual': 1223,\n",
              "   'correct': 296,\n",
              "   'incorrect': 0,\n",
              "   'missed': 298,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.24202780049059688,\n",
              "   'recall': 0.4983164983164983,\n",
              "   'spurious': 927},\n",
              "  'strict': {'actual': 1223,\n",
              "   'correct': 247,\n",
              "   'incorrect': 49,\n",
              "   'missed': 298,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.2019623875715454,\n",
              "   'recall': 0.4158249158249158,\n",
              "   'spurious': 927}},\n",
              " {'LOC': {'ent_type': {'actual': 259,\n",
              "    'correct': 131,\n",
              "    'incorrect': 34,\n",
              "    'missed': 56,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.5057915057915058,\n",
              "    'recall': 0.5927601809954751,\n",
              "    'spurious': 94},\n",
              "   'exact': {'actual': 259,\n",
              "    'correct': 165,\n",
              "    'incorrect': 0,\n",
              "    'missed': 56,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.637065637065637,\n",
              "    'recall': 0.746606334841629,\n",
              "    'spurious': 94},\n",
              "   'partial': {'actual': 259,\n",
              "    'correct': 165,\n",
              "    'incorrect': 0,\n",
              "    'missed': 56,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.637065637065637,\n",
              "    'recall': 0.746606334841629,\n",
              "    'spurious': 94},\n",
              "   'strict': {'actual': 259,\n",
              "    'correct': 131,\n",
              "    'incorrect': 34,\n",
              "    'missed': 56,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.5057915057915058,\n",
              "    'recall': 0.5927601809954751,\n",
              "    'spurious': 94}},\n",
              "  'ORG': {'ent_type': {'actual': 580,\n",
              "    'correct': 35,\n",
              "    'incorrect': 8,\n",
              "    'missed': 86,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.0603448275862069,\n",
              "    'recall': 0.2713178294573643,\n",
              "    'spurious': 537},\n",
              "   'exact': {'actual': 580,\n",
              "    'correct': 43,\n",
              "    'incorrect': 0,\n",
              "    'missed': 86,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.07413793103448275,\n",
              "    'recall': 0.3333333333333333,\n",
              "    'spurious': 537},\n",
              "   'partial': {'actual': 580,\n",
              "    'correct': 43,\n",
              "    'incorrect': 0,\n",
              "    'missed': 86,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.07413793103448275,\n",
              "    'recall': 0.3333333333333333,\n",
              "    'spurious': 537},\n",
              "   'strict': {'actual': 580,\n",
              "    'correct': 35,\n",
              "    'incorrect': 8,\n",
              "    'missed': 86,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.0603448275862069,\n",
              "    'recall': 0.2713178294573643,\n",
              "    'spurious': 537}},\n",
              "  'PER': {'ent_type': {'actual': 384,\n",
              "    'correct': 81,\n",
              "    'incorrect': 7,\n",
              "    'missed': 156,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.2109375,\n",
              "    'recall': 0.3319672131147541,\n",
              "    'spurious': 296},\n",
              "   'exact': {'actual': 384,\n",
              "    'correct': 88,\n",
              "    'incorrect': 0,\n",
              "    'missed': 156,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.22916666666666666,\n",
              "    'recall': 0.36065573770491804,\n",
              "    'spurious': 296},\n",
              "   'partial': {'actual': 384,\n",
              "    'correct': 88,\n",
              "    'incorrect': 0,\n",
              "    'missed': 156,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.22916666666666666,\n",
              "    'recall': 0.36065573770491804,\n",
              "    'spurious': 296},\n",
              "   'strict': {'actual': 384,\n",
              "    'correct': 81,\n",
              "    'incorrect': 7,\n",
              "    'missed': 156,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.2109375,\n",
              "    'recall': 0.3319672131147541,\n",
              "    'spurious': 296}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NAxp1pJXSHse"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k7XB5Dr8B7qo"
      },
      "source": [
        "### Our Model: using fine tunning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3MejRQTdCSBe"
      },
      "source": [
        "from transformers import TFAutoModelForTokenClassification"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iv491WXyCAFv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330,
          "referenced_widgets": [
            "9d0b2696074b46879f6f8132faa6b8ec",
            "7d23502f535543bfbbdbe0f685f6dfa1",
            "061636e8fc294d4ba57ff15477a97a72",
            "5de62407aead4332a7c871c2afb65fa2",
            "9348835bd95a40c39472f2b54dc26ddb",
            "80d23c4a39524334956020a30cd23b44",
            "ecd1fb04eb7a4bb6937268d1a66fcbb3",
            "645773f0eabb4dd49d27f30362181d09",
            "93f73fabe300408183d8fc387a187981",
            "bd7422b62304400cad68767f3de43e8d",
            "a134ab2d1820490a9c67ab3580236c79",
            "2f46ae8c30774cbd8854a66407498530",
            "c9a96717370d498e84bded7e7f6f1e53",
            "a1e902506782420984948151fea41ce4",
            "e8d8701f8bbf4556983a410384b695e4",
            "5d73533926084e99adea6e65f8276e46",
            "7b64322e3ffc49e2b54bb60d39dec20f",
            "209333e1db8148468d5982306950dc57",
            "f32144cfd095404c9122cbbd25559ce1",
            "45ffee06ed2e472d8af41b9a4455ce6b",
            "a19d00d6f672415682e2f3e3d259dcd6",
            "20907867e0994ca8a222d52621690347",
            "74a0f24dfaef4d5b8ae1ff7fd629ab60",
            "974cf4dd7c89413f956b2112b9628006",
            "19c90bda858443dcbae7f6ba576039bb",
            "b5b7b968348040bd86516dabbd087edc",
            "9217de938fad43a4ac385618ee069391",
            "81d15bda83db4da4a4d3504c1f3f9106",
            "845c625486fc415c97974b645e1884f6",
            "b35590632a1d40069f17233af292579a",
            "179b35e60d364fd1a1c5cd775ca47583",
            "6470a038a522450ab8151c98fa72f47b",
            "03e590ff0ce54c3bb70497b98a343777",
            "09662437a6af4643bd046400efca62c5",
            "b88be82d8dd04678a86fa740e1ad17fc",
            "bde2258c8bb04769a7d2b3832dd6ecea",
            "a3fc61cfa9e742208c9d9221238ca0f8",
            "5be4c8b7c7ca40e2b78c6b5db1331c53",
            "7dac4f5a990a4c18b3b6b1499ae3d855",
            "b4161a8e30ea40c0a0088235aad3af60"
          ]
        },
        "outputId": "cf56b61c-a94e-4db1-e4e4-ea0c7bbce587"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"overfit/twiner-bert-base\")\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\"overfit/twiner-bert-base\")\n",
        "twiner_seq = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9d0b2696074b46879f6f8132faa6b8ec",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1124.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93f73fabe300408183d8fc387a187981",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7b64322e3ffc49e2b54bb60d39dec20f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "19c90bda858443dcbae7f6ba576039bb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=354.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "03e590ff0ce54c3bb70497b98a343777",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=649311328.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForTokenClassification.\n",
            "\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at overfit/twiner-bert-base.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RskGGdFgC7oW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93763b04-0911-4227-9071-65e0715c2a41"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"B-EVE\",\n",
        "  \"B-LOC\": \"B-LOC\",\n",
        "  \"B-ORG\": \"B-ORG\",\n",
        "  \"B-PER\": \"B-PER\",\n",
        "  \"B-POG\": \"B_POG\",\n",
        "  \"B-NAT\": \"B-NAT\",\n",
        "  \"I-EVE\": \"I-EVE\",\n",
        "  \"I-LOC\": \"I-LOC\",\n",
        "  \"I-ORG\": \"I-ORG\",\n",
        "  \"I-PER\": \"I-PER\",\n",
        "  \"I-POG\": \"I_POG\",\n",
        "  \"I-NAT\": \"I-NAT\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts, tags, twiner_seq, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-16 09:42:16 tensorflow WARNING: From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-16 09:43:35 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 631,\n",
              "   'correct': 516,\n",
              "   'incorrect': 21,\n",
              "   'missed': 71,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.8177496038034865,\n",
              "   'recall': 0.8486842105263158,\n",
              "   'spurious': 94},\n",
              "  'exact': {'actual': 631,\n",
              "   'correct': 517,\n",
              "   'incorrect': 20,\n",
              "   'missed': 71,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.8193343898573693,\n",
              "   'recall': 0.850328947368421,\n",
              "   'spurious': 94},\n",
              "  'partial': {'actual': 611,\n",
              "   'correct': 517,\n",
              "   'incorrect': 0,\n",
              "   'missed': 71,\n",
              "   'partial': 20,\n",
              "   'possible': 608,\n",
              "   'precision': 0.8625204582651391,\n",
              "   'recall': 0.8667763157894737,\n",
              "   'spurious': 94},\n",
              "  'strict': {'actual': 631,\n",
              "   'correct': 498,\n",
              "   'incorrect': 39,\n",
              "   'missed': 71,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7892234548335975,\n",
              "   'recall': 0.819078947368421,\n",
              "   'spurious': 94}},\n",
              " {'EVE': {'ent_type': {'actual': 13,\n",
              "    'correct': 7,\n",
              "    'incorrect': 0,\n",
              "    'missed': 7,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5384615384615384,\n",
              "    'recall': 0.5,\n",
              "    'spurious': 6},\n",
              "   'exact': {'actual': 13,\n",
              "    'correct': 4,\n",
              "    'incorrect': 3,\n",
              "    'missed': 7,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.3076923076923077,\n",
              "    'recall': 0.2857142857142857,\n",
              "    'spurious': 6},\n",
              "   'partial': {'actual': 10,\n",
              "    'correct': 4,\n",
              "    'incorrect': 0,\n",
              "    'missed': 7,\n",
              "    'partial': 3,\n",
              "    'possible': 14,\n",
              "    'precision': 0.55,\n",
              "    'recall': 0.39285714285714285,\n",
              "    'spurious': 6},\n",
              "   'strict': {'actual': 13,\n",
              "    'correct': 4,\n",
              "    'incorrect': 3,\n",
              "    'missed': 7,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.3076923076923077,\n",
              "    'recall': 0.2857142857142857,\n",
              "    'spurious': 6}},\n",
              "  'LOC': {'ent_type': {'actual': 224,\n",
              "    'correct': 192,\n",
              "    'incorrect': 10,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8571428571428571,\n",
              "    'recall': 0.8687782805429864,\n",
              "    'spurious': 22},\n",
              "   'exact': {'actual': 224,\n",
              "    'correct': 198,\n",
              "    'incorrect': 4,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8839285714285714,\n",
              "    'recall': 0.8959276018099548,\n",
              "    'spurious': 22},\n",
              "   'partial': {'actual': 220,\n",
              "    'correct': 198,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 4,\n",
              "    'possible': 221,\n",
              "    'precision': 0.9090909090909091,\n",
              "    'recall': 0.9049773755656109,\n",
              "    'spurious': 22},\n",
              "   'strict': {'actual': 224,\n",
              "    'correct': 189,\n",
              "    'incorrect': 13,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.84375,\n",
              "    'recall': 0.8552036199095022,\n",
              "    'spurious': 22}},\n",
              "  'ORG': {'ent_type': {'actual': 131,\n",
              "    'correct': 90,\n",
              "    'incorrect': 8,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6870229007633588,\n",
              "    'recall': 0.6976744186046512,\n",
              "    'spurious': 33},\n",
              "   'exact': {'actual': 131,\n",
              "    'correct': 88,\n",
              "    'incorrect': 10,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6717557251908397,\n",
              "    'recall': 0.6821705426356589,\n",
              "    'spurious': 33},\n",
              "   'partial': {'actual': 121,\n",
              "    'correct': 88,\n",
              "    'incorrect': 0,\n",
              "    'missed': 31,\n",
              "    'partial': 10,\n",
              "    'possible': 129,\n",
              "    'precision': 0.768595041322314,\n",
              "    'recall': 0.7209302325581395,\n",
              "    'spurious': 33},\n",
              "   'strict': {'actual': 131,\n",
              "    'correct': 81,\n",
              "    'incorrect': 17,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6183206106870229,\n",
              "    'recall': 0.627906976744186,\n",
              "    'spurious': 33}},\n",
              "  'PER': {'ent_type': {'actual': 263,\n",
              "    'correct': 227,\n",
              "    'incorrect': 3,\n",
              "    'missed': 14,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8631178707224335,\n",
              "    'recall': 0.930327868852459,\n",
              "    'spurious': 33},\n",
              "   'exact': {'actual': 263,\n",
              "    'correct': 227,\n",
              "    'incorrect': 3,\n",
              "    'missed': 14,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8631178707224335,\n",
              "    'recall': 0.930327868852459,\n",
              "    'spurious': 33},\n",
              "   'partial': {'actual': 260,\n",
              "    'correct': 227,\n",
              "    'incorrect': 0,\n",
              "    'missed': 14,\n",
              "    'partial': 3,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8788461538461538,\n",
              "    'recall': 0.9364754098360656,\n",
              "    'spurious': 33},\n",
              "   'strict': {'actual': 263,\n",
              "    'correct': 224,\n",
              "    'incorrect': 6,\n",
              "    'missed': 14,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8517110266159695,\n",
              "    'recall': 0.9180327868852459,\n",
              "    'spurious': 33}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NiKdE3xuF3V6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aea1c014-0f07-4cdf-a873-fb1496e515fe"
      },
      "source": [
        "y_true, y_perd = align_prediction(texts, tags, twiner_seq, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE', 'POG', 'NAT'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-16 09:52:43 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 689,\n",
              "   'correct': 549,\n",
              "   'incorrect': 30,\n",
              "   'missed': 81,\n",
              "   'partial': 0,\n",
              "   'possible': 660,\n",
              "   'precision': 0.7968069666182874,\n",
              "   'recall': 0.8318181818181818,\n",
              "   'spurious': 110},\n",
              "  'exact': {'actual': 689,\n",
              "   'correct': 558,\n",
              "   'incorrect': 21,\n",
              "   'missed': 81,\n",
              "   'partial': 0,\n",
              "   'possible': 660,\n",
              "   'precision': 0.8098693759071117,\n",
              "   'recall': 0.8454545454545455,\n",
              "   'spurious': 110},\n",
              "  'partial': {'actual': 668,\n",
              "   'correct': 558,\n",
              "   'incorrect': 0,\n",
              "   'missed': 81,\n",
              "   'partial': 21,\n",
              "   'possible': 660,\n",
              "   'precision': 0.8510479041916168,\n",
              "   'recall': 0.8613636363636363,\n",
              "   'spurious': 110},\n",
              "  'strict': {'actual': 689,\n",
              "   'correct': 530,\n",
              "   'incorrect': 49,\n",
              "   'missed': 81,\n",
              "   'partial': 0,\n",
              "   'possible': 660,\n",
              "   'precision': 0.7692307692307693,\n",
              "   'recall': 0.803030303030303,\n",
              "   'spurious': 110}},\n",
              " {'EVE': {'ent_type': {'actual': 13,\n",
              "    'correct': 7,\n",
              "    'incorrect': 0,\n",
              "    'missed': 7,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.5384615384615384,\n",
              "    'recall': 0.5,\n",
              "    'spurious': 6},\n",
              "   'exact': {'actual': 13,\n",
              "    'correct': 4,\n",
              "    'incorrect': 3,\n",
              "    'missed': 7,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.3076923076923077,\n",
              "    'recall': 0.2857142857142857,\n",
              "    'spurious': 6},\n",
              "   'partial': {'actual': 10,\n",
              "    'correct': 4,\n",
              "    'incorrect': 0,\n",
              "    'missed': 7,\n",
              "    'partial': 3,\n",
              "    'possible': 14,\n",
              "    'precision': 0.55,\n",
              "    'recall': 0.39285714285714285,\n",
              "    'spurious': 6},\n",
              "   'strict': {'actual': 13,\n",
              "    'correct': 4,\n",
              "    'incorrect': 3,\n",
              "    'missed': 7,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.3076923076923077,\n",
              "    'recall': 0.2857142857142857,\n",
              "    'spurious': 6}},\n",
              "  'LOC': {'ent_type': {'actual': 223,\n",
              "    'correct': 192,\n",
              "    'incorrect': 10,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8609865470852018,\n",
              "    'recall': 0.8687782805429864,\n",
              "    'spurious': 21},\n",
              "   'exact': {'actual': 223,\n",
              "    'correct': 198,\n",
              "    'incorrect': 4,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8878923766816144,\n",
              "    'recall': 0.8959276018099548,\n",
              "    'spurious': 21},\n",
              "   'partial': {'actual': 219,\n",
              "    'correct': 198,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 4,\n",
              "    'possible': 221,\n",
              "    'precision': 0.91324200913242,\n",
              "    'recall': 0.9049773755656109,\n",
              "    'spurious': 21},\n",
              "   'strict': {'actual': 223,\n",
              "    'correct': 189,\n",
              "    'incorrect': 13,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8475336322869955,\n",
              "    'recall': 0.8552036199095022,\n",
              "    'spurious': 21}},\n",
              "  'NAT': {'ent_type': {'actual': 38,\n",
              "    'correct': 23,\n",
              "    'incorrect': 2,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.6052631578947368,\n",
              "    'recall': 0.7666666666666667,\n",
              "    'spurious': 13},\n",
              "   'exact': {'actual': 38,\n",
              "    'correct': 25,\n",
              "    'incorrect': 0,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.6578947368421053,\n",
              "    'recall': 0.8333333333333334,\n",
              "    'spurious': 13},\n",
              "   'partial': {'actual': 38,\n",
              "    'correct': 25,\n",
              "    'incorrect': 0,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.6578947368421053,\n",
              "    'recall': 0.8333333333333334,\n",
              "    'spurious': 13},\n",
              "   'strict': {'actual': 38,\n",
              "    'correct': 23,\n",
              "    'incorrect': 2,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.6052631578947368,\n",
              "    'recall': 0.7666666666666667,\n",
              "    'spurious': 13}},\n",
              "  'ORG': {'ent_type': {'actual': 130,\n",
              "    'correct': 90,\n",
              "    'incorrect': 8,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6923076923076923,\n",
              "    'recall': 0.6976744186046512,\n",
              "    'spurious': 32},\n",
              "   'exact': {'actual': 130,\n",
              "    'correct': 88,\n",
              "    'incorrect': 10,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.676923076923077,\n",
              "    'recall': 0.6821705426356589,\n",
              "    'spurious': 32},\n",
              "   'partial': {'actual': 120,\n",
              "    'correct': 88,\n",
              "    'incorrect': 0,\n",
              "    'missed': 31,\n",
              "    'partial': 10,\n",
              "    'possible': 129,\n",
              "    'precision': 0.775,\n",
              "    'recall': 0.7209302325581395,\n",
              "    'spurious': 32},\n",
              "   'strict': {'actual': 130,\n",
              "    'correct': 81,\n",
              "    'incorrect': 17,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6230769230769231,\n",
              "    'recall': 0.627906976744186,\n",
              "    'spurious': 32}},\n",
              "  'PER': {'ent_type': {'actual': 260,\n",
              "    'correct': 227,\n",
              "    'incorrect': 3,\n",
              "    'missed': 14,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8730769230769231,\n",
              "    'recall': 0.930327868852459,\n",
              "    'spurious': 30},\n",
              "   'exact': {'actual': 260,\n",
              "    'correct': 227,\n",
              "    'incorrect': 3,\n",
              "    'missed': 14,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8730769230769231,\n",
              "    'recall': 0.930327868852459,\n",
              "    'spurious': 30},\n",
              "   'partial': {'actual': 257,\n",
              "    'correct': 227,\n",
              "    'incorrect': 0,\n",
              "    'missed': 14,\n",
              "    'partial': 3,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8891050583657587,\n",
              "    'recall': 0.9364754098360656,\n",
              "    'spurious': 30},\n",
              "   'strict': {'actual': 260,\n",
              "    'correct': 224,\n",
              "    'incorrect': 6,\n",
              "    'missed': 14,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8615384615384616,\n",
              "    'recall': 0.9180327868852459,\n",
              "    'spurious': 30}},\n",
              "  'POG': {'ent_type': {'actual': 25,\n",
              "    'correct': 10,\n",
              "    'incorrect': 7,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.45454545454545453,\n",
              "    'spurious': 8},\n",
              "   'exact': {'actual': 25,\n",
              "    'correct': 16,\n",
              "    'incorrect': 1,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.64,\n",
              "    'recall': 0.7272727272727273,\n",
              "    'spurious': 8},\n",
              "   'partial': {'actual': 24,\n",
              "    'correct': 16,\n",
              "    'incorrect': 0,\n",
              "    'missed': 5,\n",
              "    'partial': 1,\n",
              "    'possible': 22,\n",
              "    'precision': 0.6875,\n",
              "    'recall': 0.75,\n",
              "    'spurious': 8},\n",
              "   'strict': {'actual': 25,\n",
              "    'correct': 9,\n",
              "    'incorrect': 8,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.36,\n",
              "    'recall': 0.4090909090909091,\n",
              "    'spurious': 8}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uPfRvxZRD0Ax",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "724c2102-05fb-4d7e-c874-43d812550c5b"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"O\",\n",
        "  \"B-LOC\": \"B_LOC\",\n",
        "  \"B-ORG\": \"B_ORG\",\n",
        "  \"B-PER\": \"B_PER\",\n",
        "  \"B-POG\": \"O\",\n",
        "  \"B-NAT\": \"O\",\n",
        "  \"I-EVE\": \"O\",\n",
        "  \"I-LOC\": \"I_LOC\",\n",
        "  \"I-ORG\": \"I_ORG\",\n",
        "  \"I-PER\": \"I_PER\",\n",
        "  \"I-POG\": \"O\",\n",
        "  \"I-NAT\": \"O\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts_peyma, tags_peyma, twiner_seq, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 14:59:30 tensorflow WARNING: From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 15:02:10 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 850,\n",
              "   'correct': 684,\n",
              "   'incorrect': 14,\n",
              "   'missed': 147,\n",
              "   'partial': 0,\n",
              "   'possible': 845,\n",
              "   'precision': 0.8047058823529412,\n",
              "   'recall': 0.8094674556213017,\n",
              "   'spurious': 152},\n",
              "  'exact': {'actual': 850,\n",
              "   'correct': 643,\n",
              "   'incorrect': 55,\n",
              "   'missed': 147,\n",
              "   'partial': 0,\n",
              "   'possible': 845,\n",
              "   'precision': 0.7564705882352941,\n",
              "   'recall': 0.7609467455621302,\n",
              "   'spurious': 152},\n",
              "  'partial': {'actual': 795,\n",
              "   'correct': 643,\n",
              "   'incorrect': 0,\n",
              "   'missed': 147,\n",
              "   'partial': 55,\n",
              "   'possible': 845,\n",
              "   'precision': 0.8433962264150944,\n",
              "   'recall': 0.7934911242603551,\n",
              "   'spurious': 152},\n",
              "  'strict': {'actual': 850,\n",
              "   'correct': 638,\n",
              "   'incorrect': 60,\n",
              "   'missed': 147,\n",
              "   'partial': 0,\n",
              "   'possible': 845,\n",
              "   'precision': 0.7505882352941177,\n",
              "   'recall': 0.7550295857988165,\n",
              "   'spurious': 152}},\n",
              " {'LOC': {'ent_type': {'actual': 318,\n",
              "    'correct': 249,\n",
              "    'incorrect': 3,\n",
              "    'missed': 54,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.7830188679245284,\n",
              "    'recall': 0.8137254901960784,\n",
              "    'spurious': 66},\n",
              "   'exact': {'actual': 318,\n",
              "    'correct': 242,\n",
              "    'incorrect': 10,\n",
              "    'missed': 54,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.7610062893081762,\n",
              "    'recall': 0.7908496732026143,\n",
              "    'spurious': 66},\n",
              "   'partial': {'actual': 308,\n",
              "    'correct': 242,\n",
              "    'incorrect': 0,\n",
              "    'missed': 54,\n",
              "    'partial': 10,\n",
              "    'possible': 306,\n",
              "    'precision': 0.801948051948052,\n",
              "    'recall': 0.8071895424836601,\n",
              "    'spurious': 66},\n",
              "   'strict': {'actual': 318,\n",
              "    'correct': 241,\n",
              "    'incorrect': 11,\n",
              "    'missed': 54,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.7578616352201258,\n",
              "    'recall': 0.7875816993464052,\n",
              "    'spurious': 66}},\n",
              "  'ORG': {'ent_type': {'actual': 286,\n",
              "    'correct': 215,\n",
              "    'incorrect': 9,\n",
              "    'missed': 88,\n",
              "    'partial': 0,\n",
              "    'possible': 312,\n",
              "    'precision': 0.7517482517482518,\n",
              "    'recall': 0.6891025641025641,\n",
              "    'spurious': 62},\n",
              "   'exact': {'actual': 286,\n",
              "    'correct': 182,\n",
              "    'incorrect': 42,\n",
              "    'missed': 88,\n",
              "    'partial': 0,\n",
              "    'possible': 312,\n",
              "    'precision': 0.6363636363636364,\n",
              "    'recall': 0.5833333333333334,\n",
              "    'spurious': 62},\n",
              "   'partial': {'actual': 244,\n",
              "    'correct': 182,\n",
              "    'incorrect': 0,\n",
              "    'missed': 88,\n",
              "    'partial': 42,\n",
              "    'possible': 312,\n",
              "    'precision': 0.8319672131147541,\n",
              "    'recall': 0.6506410256410257,\n",
              "    'spurious': 62},\n",
              "   'strict': {'actual': 286,\n",
              "    'correct': 180,\n",
              "    'incorrect': 44,\n",
              "    'missed': 88,\n",
              "    'partial': 0,\n",
              "    'possible': 312,\n",
              "    'precision': 0.6293706293706294,\n",
              "    'recall': 0.5769230769230769,\n",
              "    'spurious': 62}},\n",
              "  'PER': {'ent_type': {'actual': 246,\n",
              "    'correct': 220,\n",
              "    'incorrect': 2,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.8943089430894309,\n",
              "    'recall': 0.9691629955947136,\n",
              "    'spurious': 24},\n",
              "   'exact': {'actual': 246,\n",
              "    'correct': 219,\n",
              "    'incorrect': 3,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.8902439024390244,\n",
              "    'recall': 0.9647577092511013,\n",
              "    'spurious': 24},\n",
              "   'partial': {'actual': 243,\n",
              "    'correct': 219,\n",
              "    'incorrect': 0,\n",
              "    'missed': 5,\n",
              "    'partial': 3,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9074074074074074,\n",
              "    'recall': 0.9713656387665198,\n",
              "    'spurious': 24},\n",
              "   'strict': {'actual': 246,\n",
              "    'correct': 217,\n",
              "    'incorrect': 5,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.8821138211382114,\n",
              "    'recall': 0.9559471365638766,\n",
              "    'spurious': 24}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tT0D23USxBMk"
      },
      "source": [
        "### Our Model using MTL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sunGNqVExHuq"
      },
      "source": [
        "from transformers import TFAutoModelForTokenClassification"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSpH9G2UxDE_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "cc8ae7f518b847e199b601717debd49c",
            "671dbf7ab46a43da96f80a325d5df9de",
            "14aecdd86ae2440da90dd9e604fbf599",
            "2b981119248b4f838643be99a2bed3d0",
            "479e907d7c14401a85ba89345c6dde13",
            "f636d79955674c2b89c1a243987fc71e",
            "a8fb34c2c0884eb58285666cda1279cc",
            "26488cebb72e47f3ba1bd8fbabb12fad",
            "9dd5593e8a99449d8d3fcaed7052bfc9",
            "94b2c4839a964a2b9b9313545ba93a82",
            "d186cd5f37fe49b3a568ea81a81b7bad",
            "253068a8570e48efa263cc14848ec8c0",
            "a883103d05f844ae95cd3ef2b504bcf8",
            "92438dc593ad464780696aeeb6a2c55d",
            "55159a4c579e4ea09bc8ce5ae704ebf2",
            "45aa8fb090b741baa05b1433b943c73a",
            "3e2213f76bb74c09b228641f2a25d105",
            "14eda72734cc47eda9f2215760f4c5a5",
            "2cc642ba0da5482fb9adaefa266404ba",
            "f6a2f7d1dd5041439e2ff97d1e83e354",
            "e2e115907bf646edb88eb4613eb225f3",
            "a082ddb6bdfe48fbac4bd9449707a044",
            "f76786ad2ea34d5cacfd7e4042135d18",
            "1ec8f2a7bd224364bfe80e93b62f8b59",
            "367de1b57efd48608c647a04dac8e34d",
            "0ba6367897a241b4b31980077f442a1d",
            "209f15203ddb48139c047ddfbe013def",
            "41b22596ecfb4d1fa50fe8a14502c85c",
            "824212d44eb1483b9d8843bbd6d1999c",
            "3350d0511e7846fc8978b6bbe4567d1b",
            "c1f566b5e5db4e7f96b64d7a67a7a91a",
            "0058a66157584b6698434b5481d06fc6",
            "1a4aa8a86b88416492cd7cc14b1aeab8",
            "5db5d64647d94e7fa0c4a0a4f7d700af",
            "1f16ffc3782d442bafda0c6c42f59a7a",
            "372eca8f78fd4e52a15ef0bb73d0182f",
            "e11d45a97f784ff6a88ca708b9a09bf9",
            "42db72165339459c8ffae5390ba03f0c",
            "1085049fb01e4098bf681d26d49b59f2",
            "5bea16934e16416aba618cf38d4f50b5"
          ]
        },
        "outputId": "7418903c-1c89-4730-957c-c9630fbfe1dd"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "twiner_mtl = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 17:52:11 filelock DEBUG: Attempting to acquire lock 140370185980624 on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-17 17:52:11 filelock INFO: Lock 140370185980624 acquired on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 1124\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cc8ae7f518b847e199b601717debd49c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1124.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:11 filelock DEBUG: Attempting to release lock 140370185980624 on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-17 17:52:11 filelock INFO: Lock 140370185980624 released on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-17 17:52:11 filelock DEBUG: Attempting to acquire lock 140370181925136 on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 17:52:11 filelock INFO: Lock 140370181925136 acquired on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:11 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/vocab.txt HTTP/1.1\" 200 1215509\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9dd5593e8a99449d8d3fcaed7052bfc9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:12 filelock DEBUG: Attempting to release lock 140370181925136 on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 17:52:12 filelock INFO: Lock 140370181925136 released on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-17 17:52:12 filelock DEBUG: Attempting to acquire lock 140370185980624 on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 17:52:12 filelock INFO: Lock 140370185980624 acquired on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/special_tokens_map.json HTTP/1.1\" 200 112\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e2213f76bb74c09b228641f2a25d105",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:12 filelock DEBUG: Attempting to release lock 140370185980624 on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 17:52:12 filelock INFO: Lock 140370185980624 released on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 17:52:12 filelock DEBUG: Attempting to acquire lock 140372767008848 on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-17 17:52:12 filelock INFO: Lock 140372767008848 acquired on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/tokenizer_config.json HTTP/1.1\" 200 354\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "367de1b57efd48608c647a04dac8e34d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=354.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:12 filelock DEBUG: Attempting to release lock 140372767008848 on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-17 17:52:12 filelock INFO: Lock 140372767008848 released on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:13 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:13 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 17:52:13 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 17:52:13 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tf_model.h5 HTTP/1.1\" 302 0\n",
            "2021-04-17 17:52:13 filelock DEBUG: Attempting to acquire lock 140370181635408 on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n",
            "2021-04-17 17:52:13 filelock INFO: Lock 140370181635408 acquired on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n",
            "2021-04-17 17:52:13 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-17 17:52:13 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/2d70b4698a3d6ebcf87ee24ed94b0d5ebcbd9bc7890ae991171c3f488675f128 HTTP/1.1\" 200 649311328\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1a4aa8a86b88416492cd7cc14b1aeab8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=649311328.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:52:28 filelock DEBUG: Attempting to release lock 140370181635408 on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n",
            "2021-04-17 17:52:28 filelock INFO: Lock 140370181635408 released on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForTokenClassification.\n",
            "\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at overfit/twiner-bert-base-mtl.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aaJkFJFQxgel",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c89c436a-58a5-4381-977d-32c5a7f53f63"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"B-EVE\",\n",
        "  \"B-LOC\": \"B-LOC\",\n",
        "  \"B-ORG\": \"B-ORG\",\n",
        "  \"B-PER\": \"B-PER\",\n",
        "  \"B-POG\": \"B-POG\",\n",
        "  \"B-NAT\": \"B-NAT\",\n",
        "  \"I-EVE\": \"I-EVE\",\n",
        "  \"I-LOC\": \"I-LOC\",\n",
        "  \"I-ORG\": \"I-ORG\",\n",
        "  \"I-PER\": \"I-PER\",\n",
        "  \"I-POG\": \"I_POG\",\n",
        "  \"I-NAT\": \"I-NAT\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts, tags, twiner_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:16:29 tensorflow WARNING: From /usr/local/lib/python3.7/dist-packages/transformers/pipelines/token_classification.py:158: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.identity instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:17:59 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 630,\n",
              "   'correct': 514,\n",
              "   'incorrect': 15,\n",
              "   'missed': 79,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.8158730158730159,\n",
              "   'recall': 0.8453947368421053,\n",
              "   'spurious': 101},\n",
              "  'exact': {'actual': 630,\n",
              "   'correct': 509,\n",
              "   'incorrect': 20,\n",
              "   'missed': 79,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.807936507936508,\n",
              "   'recall': 0.837171052631579,\n",
              "   'spurious': 101},\n",
              "  'partial': {'actual': 610,\n",
              "   'correct': 509,\n",
              "   'incorrect': 0,\n",
              "   'missed': 79,\n",
              "   'partial': 20,\n",
              "   'possible': 608,\n",
              "   'precision': 0.8508196721311475,\n",
              "   'recall': 0.8536184210526315,\n",
              "   'spurious': 101},\n",
              "  'strict': {'actual': 630,\n",
              "   'correct': 496,\n",
              "   'incorrect': 33,\n",
              "   'missed': 79,\n",
              "   'partial': 0,\n",
              "   'possible': 608,\n",
              "   'precision': 0.7873015873015873,\n",
              "   'recall': 0.8157894736842105,\n",
              "   'spurious': 101}},\n",
              " {'EVE': {'ent_type': {'actual': 15,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.6666666666666666,\n",
              "    'recall': 0.7142857142857143,\n",
              "    'spurious': 5},\n",
              "   'exact': {'actual': 15,\n",
              "    'correct': 6,\n",
              "    'incorrect': 4,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.42857142857142855,\n",
              "    'spurious': 5},\n",
              "   'partial': {'actual': 11,\n",
              "    'correct': 6,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 4,\n",
              "    'possible': 14,\n",
              "    'precision': 0.7272727272727273,\n",
              "    'recall': 0.5714285714285714,\n",
              "    'spurious': 5},\n",
              "   'strict': {'actual': 15,\n",
              "    'correct': 6,\n",
              "    'incorrect': 4,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.42857142857142855,\n",
              "    'spurious': 5}},\n",
              "  'LOC': {'ent_type': {'actual': 230,\n",
              "    'correct': 189,\n",
              "    'incorrect': 6,\n",
              "    'missed': 26,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8217391304347826,\n",
              "    'recall': 0.8552036199095022,\n",
              "    'spurious': 35},\n",
              "   'exact': {'actual': 230,\n",
              "    'correct': 190,\n",
              "    'incorrect': 5,\n",
              "    'missed': 26,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8260869565217391,\n",
              "    'recall': 0.8597285067873304,\n",
              "    'spurious': 35},\n",
              "   'partial': {'actual': 225,\n",
              "    'correct': 190,\n",
              "    'incorrect': 0,\n",
              "    'missed': 26,\n",
              "    'partial': 5,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8555555555555555,\n",
              "    'recall': 0.8710407239819005,\n",
              "    'spurious': 35},\n",
              "   'strict': {'actual': 230,\n",
              "    'correct': 185,\n",
              "    'incorrect': 10,\n",
              "    'missed': 26,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8043478260869565,\n",
              "    'recall': 0.8371040723981901,\n",
              "    'spurious': 35}},\n",
              "  'ORG': {'ent_type': {'actual': 138,\n",
              "    'correct': 93,\n",
              "    'incorrect': 5,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6739130434782609,\n",
              "    'recall': 0.7209302325581395,\n",
              "    'spurious': 40},\n",
              "   'exact': {'actual': 138,\n",
              "    'correct': 91,\n",
              "    'incorrect': 7,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6594202898550725,\n",
              "    'recall': 0.7054263565891473,\n",
              "    'spurious': 40},\n",
              "   'partial': {'actual': 131,\n",
              "    'correct': 91,\n",
              "    'incorrect': 0,\n",
              "    'missed': 31,\n",
              "    'partial': 7,\n",
              "    'possible': 129,\n",
              "    'precision': 0.7213740458015268,\n",
              "    'recall': 0.7325581395348837,\n",
              "    'spurious': 40},\n",
              "   'strict': {'actual': 138,\n",
              "    'correct': 86,\n",
              "    'incorrect': 12,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6231884057971014,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 40}},\n",
              "  'PER': {'ent_type': {'actual': 247,\n",
              "    'correct': 222,\n",
              "    'incorrect': 4,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8987854251012146,\n",
              "    'recall': 0.9098360655737705,\n",
              "    'spurious': 21},\n",
              "   'exact': {'actual': 247,\n",
              "    'correct': 222,\n",
              "    'incorrect': 4,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8987854251012146,\n",
              "    'recall': 0.9098360655737705,\n",
              "    'spurious': 21},\n",
              "   'partial': {'actual': 243,\n",
              "    'correct': 222,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 4,\n",
              "    'possible': 244,\n",
              "    'precision': 0.9218106995884774,\n",
              "    'recall': 0.9180327868852459,\n",
              "    'spurious': 21},\n",
              "   'strict': {'actual': 247,\n",
              "    'correct': 219,\n",
              "    'incorrect': 7,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8866396761133604,\n",
              "    'recall': 0.8975409836065574,\n",
              "    'spurious': 21}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_SpC66cVPBa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3312126d-e7e2-48f3-9fb8-256909567169"
      },
      "source": [
        "y_true, y_perd = align_prediction(texts, tags, twiner_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE', 'POG', 'NAT'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:19:32 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 685,\n",
              "   'correct': 557,\n",
              "   'incorrect': 18,\n",
              "   'missed': 85,\n",
              "   'partial': 0,\n",
              "   'possible': 660,\n",
              "   'precision': 0.8131386861313868,\n",
              "   'recall': 0.843939393939394,\n",
              "   'spurious': 110},\n",
              "  'exact': {'actual': 685,\n",
              "   'correct': 555,\n",
              "   'incorrect': 20,\n",
              "   'missed': 85,\n",
              "   'partial': 0,\n",
              "   'possible': 660,\n",
              "   'precision': 0.8102189781021898,\n",
              "   'recall': 0.8409090909090909,\n",
              "   'spurious': 110},\n",
              "  'partial': {'actual': 665,\n",
              "   'correct': 555,\n",
              "   'incorrect': 0,\n",
              "   'missed': 85,\n",
              "   'partial': 20,\n",
              "   'possible': 660,\n",
              "   'precision': 0.849624060150376,\n",
              "   'recall': 0.8560606060606061,\n",
              "   'spurious': 110},\n",
              "  'strict': {'actual': 685,\n",
              "   'correct': 539,\n",
              "   'incorrect': 36,\n",
              "   'missed': 85,\n",
              "   'partial': 0,\n",
              "   'possible': 660,\n",
              "   'precision': 0.7868613138686131,\n",
              "   'recall': 0.8166666666666667,\n",
              "   'spurious': 110}},\n",
              " {'EVE': {'ent_type': {'actual': 15,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.6666666666666666,\n",
              "    'recall': 0.7142857142857143,\n",
              "    'spurious': 5},\n",
              "   'exact': {'actual': 15,\n",
              "    'correct': 6,\n",
              "    'incorrect': 4,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.42857142857142855,\n",
              "    'spurious': 5},\n",
              "   'partial': {'actual': 11,\n",
              "    'correct': 6,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 4,\n",
              "    'possible': 14,\n",
              "    'precision': 0.7272727272727273,\n",
              "    'recall': 0.5714285714285714,\n",
              "    'spurious': 5},\n",
              "   'strict': {'actual': 15,\n",
              "    'correct': 6,\n",
              "    'incorrect': 4,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 14,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.42857142857142855,\n",
              "    'spurious': 5}},\n",
              "  'LOC': {'ent_type': {'actual': 229,\n",
              "    'correct': 189,\n",
              "    'incorrect': 6,\n",
              "    'missed': 26,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8253275109170306,\n",
              "    'recall': 0.8552036199095022,\n",
              "    'spurious': 34},\n",
              "   'exact': {'actual': 229,\n",
              "    'correct': 190,\n",
              "    'incorrect': 5,\n",
              "    'missed': 26,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8296943231441049,\n",
              "    'recall': 0.8597285067873304,\n",
              "    'spurious': 34},\n",
              "   'partial': {'actual': 224,\n",
              "    'correct': 190,\n",
              "    'incorrect': 0,\n",
              "    'missed': 26,\n",
              "    'partial': 5,\n",
              "    'possible': 221,\n",
              "    'precision': 0.859375,\n",
              "    'recall': 0.8710407239819005,\n",
              "    'spurious': 34},\n",
              "   'strict': {'actual': 229,\n",
              "    'correct': 185,\n",
              "    'incorrect': 10,\n",
              "    'missed': 26,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.8078602620087336,\n",
              "    'recall': 0.8371040723981901,\n",
              "    'spurious': 34}},\n",
              "  'NAT': {'ent_type': {'actual': 38,\n",
              "    'correct': 28,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.7368421052631579,\n",
              "    'recall': 0.9333333333333333,\n",
              "    'spurious': 10},\n",
              "   'exact': {'actual': 38,\n",
              "    'correct': 28,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.7368421052631579,\n",
              "    'recall': 0.9333333333333333,\n",
              "    'spurious': 10},\n",
              "   'partial': {'actual': 38,\n",
              "    'correct': 28,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.7368421052631579,\n",
              "    'recall': 0.9333333333333333,\n",
              "    'spurious': 10},\n",
              "   'strict': {'actual': 38,\n",
              "    'correct': 28,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 30,\n",
              "    'precision': 0.7368421052631579,\n",
              "    'recall': 0.9333333333333333,\n",
              "    'spurious': 10}},\n",
              "  'ORG': {'ent_type': {'actual': 137,\n",
              "    'correct': 93,\n",
              "    'incorrect': 5,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6788321167883211,\n",
              "    'recall': 0.7209302325581395,\n",
              "    'spurious': 39},\n",
              "   'exact': {'actual': 137,\n",
              "    'correct': 91,\n",
              "    'incorrect': 7,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6642335766423357,\n",
              "    'recall': 0.7054263565891473,\n",
              "    'spurious': 39},\n",
              "   'partial': {'actual': 130,\n",
              "    'correct': 91,\n",
              "    'incorrect': 0,\n",
              "    'missed': 31,\n",
              "    'partial': 7,\n",
              "    'possible': 129,\n",
              "    'precision': 0.7269230769230769,\n",
              "    'recall': 0.7325581395348837,\n",
              "    'spurious': 39},\n",
              "   'strict': {'actual': 137,\n",
              "    'correct': 86,\n",
              "    'incorrect': 12,\n",
              "    'missed': 31,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6277372262773723,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 39}},\n",
              "  'PER': {'ent_type': {'actual': 246,\n",
              "    'correct': 222,\n",
              "    'incorrect': 4,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.9024390243902439,\n",
              "    'recall': 0.9098360655737705,\n",
              "    'spurious': 20},\n",
              "   'exact': {'actual': 246,\n",
              "    'correct': 222,\n",
              "    'incorrect': 4,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.9024390243902439,\n",
              "    'recall': 0.9098360655737705,\n",
              "    'spurious': 20},\n",
              "   'partial': {'actual': 242,\n",
              "    'correct': 222,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 4,\n",
              "    'possible': 244,\n",
              "    'precision': 0.9256198347107438,\n",
              "    'recall': 0.9180327868852459,\n",
              "    'spurious': 20},\n",
              "   'strict': {'actual': 246,\n",
              "    'correct': 219,\n",
              "    'incorrect': 7,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8902439024390244,\n",
              "    'recall': 0.8975409836065574,\n",
              "    'spurious': 20}},\n",
              "  'POG': {'ent_type': {'actual': 20,\n",
              "    'correct': 15,\n",
              "    'incorrect': 3,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.75,\n",
              "    'recall': 0.6818181818181818,\n",
              "    'spurious': 2},\n",
              "   'exact': {'actual': 20,\n",
              "    'correct': 18,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.9,\n",
              "    'recall': 0.8181818181818182,\n",
              "    'spurious': 2},\n",
              "   'partial': {'actual': 20,\n",
              "    'correct': 18,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.9,\n",
              "    'recall': 0.8181818181818182,\n",
              "    'spurious': 2},\n",
              "   'strict': {'actual': 20,\n",
              "    'correct': 15,\n",
              "    'incorrect': 3,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 22,\n",
              "    'precision': 0.75,\n",
              "    'recall': 0.6818181818181818,\n",
              "    'spurious': 2}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_qsbJ6xuaLZt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "528fae76-fdb3-4526-f0e6-99a17a7d994f"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"O\",\n",
        "  \"B-LOC\": \"B_LOC\",\n",
        "  \"B-ORG\": \"B_ORG\",\n",
        "  \"B-PER\": \"B_PER\",\n",
        "  \"B-POG\": \"O\",\n",
        "  \"B-NAT\": \"O\",\n",
        "  \"I-EVE\": \"O\",\n",
        "  \"I-LOC\": \"I_LOC\",\n",
        "  \"I-ORG\": \"I_ORG\",\n",
        "  \"I-PER\": \"I_PER\",\n",
        "  \"I-POG\": \"O\",\n",
        "  \"I-NAT\": \"O\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts_peyma, tags_peyma, twiner_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 17:55:32 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 847,\n",
              "   'correct': 737,\n",
              "   'incorrect': 18,\n",
              "   'missed': 87,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.8701298701298701,\n",
              "   'recall': 0.8752969121140143,\n",
              "   'spurious': 92},\n",
              "  'exact': {'actual': 847,\n",
              "   'correct': 726,\n",
              "   'incorrect': 29,\n",
              "   'missed': 87,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.8571428571428571,\n",
              "   'recall': 0.8622327790973872,\n",
              "   'spurious': 92},\n",
              "  'partial': {'actual': 818,\n",
              "   'correct': 726,\n",
              "   'incorrect': 0,\n",
              "   'missed': 87,\n",
              "   'partial': 29,\n",
              "   'possible': 842,\n",
              "   'precision': 0.9052567237163814,\n",
              "   'recall': 0.8794536817102138,\n",
              "   'spurious': 92},\n",
              "  'strict': {'actual': 847,\n",
              "   'correct': 714,\n",
              "   'incorrect': 41,\n",
              "   'missed': 87,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.8429752066115702,\n",
              "   'recall': 0.8479809976247031,\n",
              "   'spurious': 92}},\n",
              " {'LOC': {'ent_type': {'actual': 310,\n",
              "    'correct': 256,\n",
              "    'incorrect': 3,\n",
              "    'missed': 47,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8258064516129032,\n",
              "    'recall': 0.8366013071895425,\n",
              "    'spurious': 51},\n",
              "   'exact': {'actual': 310,\n",
              "    'correct': 251,\n",
              "    'incorrect': 8,\n",
              "    'missed': 47,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8096774193548387,\n",
              "    'recall': 0.8202614379084967,\n",
              "    'spurious': 51},\n",
              "   'partial': {'actual': 302,\n",
              "    'correct': 251,\n",
              "    'incorrect': 0,\n",
              "    'missed': 47,\n",
              "    'partial': 8,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8443708609271523,\n",
              "    'recall': 0.8333333333333334,\n",
              "    'spurious': 51},\n",
              "   'strict': {'actual': 310,\n",
              "    'correct': 249,\n",
              "    'incorrect': 10,\n",
              "    'missed': 47,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8032258064516129,\n",
              "    'recall': 0.8137254901960784,\n",
              "    'spurious': 51}},\n",
              "  'ORG': {'ent_type': {'actual': 303,\n",
              "    'correct': 259,\n",
              "    'incorrect': 12,\n",
              "    'missed': 38,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.8547854785478548,\n",
              "    'recall': 0.8381877022653722,\n",
              "    'spurious': 32},\n",
              "   'exact': {'actual': 303,\n",
              "    'correct': 252,\n",
              "    'incorrect': 19,\n",
              "    'missed': 38,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.8316831683168316,\n",
              "    'recall': 0.8155339805825242,\n",
              "    'spurious': 32},\n",
              "   'partial': {'actual': 284,\n",
              "    'correct': 252,\n",
              "    'incorrect': 0,\n",
              "    'missed': 38,\n",
              "    'partial': 19,\n",
              "    'possible': 309,\n",
              "    'precision': 0.920774647887324,\n",
              "    'recall': 0.8462783171521036,\n",
              "    'spurious': 32},\n",
              "   'strict': {'actual': 303,\n",
              "    'correct': 245,\n",
              "    'incorrect': 26,\n",
              "    'missed': 38,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.8085808580858086,\n",
              "    'recall': 0.7928802588996764,\n",
              "    'spurious': 32}},\n",
              "  'PER': {'ent_type': {'actual': 234,\n",
              "    'correct': 222,\n",
              "    'incorrect': 3,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9487179487179487,\n",
              "    'recall': 0.9779735682819384,\n",
              "    'spurious': 9},\n",
              "   'exact': {'actual': 234,\n",
              "    'correct': 223,\n",
              "    'incorrect': 2,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9529914529914529,\n",
              "    'recall': 0.9823788546255506,\n",
              "    'spurious': 9},\n",
              "   'partial': {'actual': 232,\n",
              "    'correct': 223,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 2,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9655172413793104,\n",
              "    'recall': 0.986784140969163,\n",
              "    'spurious': 9},\n",
              "   'strict': {'actual': 234,\n",
              "    'correct': 220,\n",
              "    'incorrect': 5,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9401709401709402,\n",
              "    'recall': 0.9691629955947136,\n",
              "    'spurious': 9}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9tcrqMkgahKz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sMnct_6geN6f"
      },
      "source": [
        "### Our Model Peyma using MTL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wp9SGRxGeP9m"
      },
      "source": [
        "from transformers import TFAutoModelForTokenClassification"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DWbq1_IeSqa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "97a4ba89598941159c9f960c2749b70e",
            "b214fe27f48d420f9d4220599d9b65e3",
            "007e48cdc9fe47cfb9959d021112ddeb",
            "0b62ccb625034afc96f3ae6542bc4498",
            "5373d106e3af44f98f77dcd6a7b4348a",
            "9d1f2f62a57e43fd8b728550758977e1",
            "c84821460e7a40af8b75134e5acf4099",
            "aac9baaad7ef48ee91fd97f7033a0ab0",
            "10ebfcfa5ea442f4b15fce3c4920a255",
            "1f4447651aeb4f4581a397a125f27875",
            "da2bd89fcb194b108b10e3e0484cfceb",
            "2f938ebf96d84f52ba9e52be9964d976",
            "12ef8cfc846d477e9305d3d639dac9a0",
            "b57b949607474828ad3c0fa3d17d07f3",
            "6d2ffe05c15445518b3904ed5cc20469",
            "ce8de109632c444b9fd407310963cc55",
            "2dafb342404a46f0b6fd87cf6382d37c",
            "1b7293a8d4f8404fa51bd46a11c5f511",
            "8bae6e70fe844fc2bee4a157d64a3c39",
            "172ba28dcef54e46850fa44f47608a7a",
            "2b3a05bb91dd4737990565873884f179",
            "0397fbdef27f477fabe0093727e9cede",
            "a23d6e99ab5e4a2ba845ab40a8b1498b",
            "673c90fe0cf24416ac4fa1a5a8d30092",
            "1a5be964f98546aabdb68b253c88e573",
            "c49abd18d56a47b693f547b72b46e3ab",
            "121121c4fe5a40e08a637fec951e9eba",
            "260f96dc84ee48d4890bbf3060c2f9dd",
            "72355c9281c04e10b342cf25132dcde3",
            "1687efa68df0407092a61fd0d7a0c0e3",
            "0cf052f4ffd04302ac209fde39f254d4",
            "ea1e969254f04cbf905fda3458eb4376",
            "c36bc42b5a3a49b0b94c9ced194525c2",
            "32988af100894ea1a7880ec516507c07",
            "7d27ee0586104b8e86556a54e79e6c60",
            "3df100c50b5c4c149772cbe7afc9fb4e",
            "db29108d394e4093a7cc4f673dd1d065",
            "c6547c32f8a441f5bb7836c505c64ba9",
            "50ca024a96ee4edd8845c31d52cbc745",
            "3c38018121a8422b93c387380af63b1b"
          ]
        },
        "outputId": "4cd4e4fc-b21f-480d-8f4a-a623a8ac9efa"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"overfit/peyma-ner-bert-base\")\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\"overfit/peyma-ner-bert-base\")\n",
        "peyma_mtl = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:04 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:04 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 10:59:04 filelock DEBUG: Attempting to acquire lock 140403316194000 on /root/.cache/huggingface/transformers/213a6ec7fc886c25c24fcecf724c386a07934a0c5df7b7d54677edd6f5784679.362070e431c5b684bc5ccefba3372fe3e7088f2b24adf061c7b45684af1dc1eb.lock\n",
            "2021-04-17 10:59:04 filelock INFO: Lock 140403316194000 acquired on /root/.cache/huggingface/transformers/213a6ec7fc886c25c24fcecf724c386a07934a0c5df7b7d54677edd6f5784679.362070e431c5b684bc5ccefba3372fe3e7088f2b24adf061c7b45684af1dc1eb.lock\n",
            "2021-04-17 10:59:04 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:04 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/peyma-ner-bert-base/resolve/main/config.json HTTP/1.1\" 200 1196\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "97a4ba89598941159c9f960c2749b70e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1196.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:04 filelock DEBUG: Attempting to release lock 140403316194000 on /root/.cache/huggingface/transformers/213a6ec7fc886c25c24fcecf724c386a07934a0c5df7b7d54677edd6f5784679.362070e431c5b684bc5ccefba3372fe3e7088f2b24adf061c7b45684af1dc1eb.lock\n",
            "2021-04-17 10:59:04 filelock INFO: Lock 140403316194000 released on /root/.cache/huggingface/transformers/213a6ec7fc886c25c24fcecf724c386a07934a0c5df7b7d54677edd6f5784679.362070e431c5b684bc5ccefba3372fe3e7088f2b24adf061c7b45684af1dc1eb.lock\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-17 10:59:05 filelock DEBUG: Attempting to acquire lock 140403524698064 on /root/.cache/huggingface/transformers/35b64b6ab37076c6af96a12b62c2effdc4d7edc3038a239694685775db44e7cb.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 10:59:05 filelock INFO: Lock 140403524698064 acquired on /root/.cache/huggingface/transformers/35b64b6ab37076c6af96a12b62c2effdc4d7edc3038a239694685775db44e7cb.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/peyma-ner-bert-base/resolve/main/vocab.txt HTTP/1.1\" 200 1215509\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "10ebfcfa5ea442f4b15fce3c4920a255",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:05 filelock DEBUG: Attempting to release lock 140403524698064 on /root/.cache/huggingface/transformers/35b64b6ab37076c6af96a12b62c2effdc4d7edc3038a239694685775db44e7cb.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 10:59:05 filelock INFO: Lock 140403524698064 released on /root/.cache/huggingface/transformers/35b64b6ab37076c6af96a12b62c2effdc4d7edc3038a239694685775db44e7cb.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-17 10:59:05 filelock DEBUG: Attempting to acquire lock 140403305755792 on /root/.cache/huggingface/transformers/d5181600882759801da7c1f669c595b7d196afdcb789dca427eb95bba0fe40ce.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 10:59:05 filelock INFO: Lock 140403305755792 acquired on /root/.cache/huggingface/transformers/d5181600882759801da7c1f669c595b7d196afdcb789dca427eb95bba0fe40ce.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/peyma-ner-bert-base/resolve/main/special_tokens_map.json HTTP/1.1\" 200 112\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2dafb342404a46f0b6fd87cf6382d37c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:05 filelock DEBUG: Attempting to release lock 140403305755792 on /root/.cache/huggingface/transformers/d5181600882759801da7c1f669c595b7d196afdcb789dca427eb95bba0fe40ce.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 10:59:05 filelock INFO: Lock 140403305755792 released on /root/.cache/huggingface/transformers/d5181600882759801da7c1f669c595b7d196afdcb789dca427eb95bba0fe40ce.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 10:59:05 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 10:59:06 filelock DEBUG: Attempting to acquire lock 140403305755792 on /root/.cache/huggingface/transformers/823b89ed1cc28a1c2ffe554fd665037ee101afe706c266d098b7baabae4eaf08.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-17 10:59:06 filelock INFO: Lock 140403305755792 acquired on /root/.cache/huggingface/transformers/823b89ed1cc28a1c2ffe554fd665037ee101afe706c266d098b7baabae4eaf08.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/peyma-ner-bert-base/resolve/main/tokenizer_config.json HTTP/1.1\" 200 354\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1a5be964f98546aabdb68b253c88e573",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=354.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:06 filelock DEBUG: Attempting to release lock 140403305755792 on /root/.cache/huggingface/transformers/823b89ed1cc28a1c2ffe554fd665037ee101afe706c266d098b7baabae4eaf08.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-17 10:59:06 filelock INFO: Lock 140403305755792 released on /root/.cache/huggingface/transformers/823b89ed1cc28a1c2ffe554fd665037ee101afe706c266d098b7baabae4eaf08.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/peyma-ner-bert-base/resolve/main/tf_model.h5 HTTP/1.1\" 302 0\n",
            "2021-04-17 10:59:06 filelock DEBUG: Attempting to acquire lock 140403321134864 on /root/.cache/huggingface/transformers/88b58bece2110e8763ccccc3f2cf3727d2759ca72eae6c299c5fb0a1b4d0b61b.c1dcf06d6a9561fe2acca9a7c9f2ec07038065d704e1984b293c6c142b53c3a8.h5.lock\n",
            "2021-04-17 10:59:06 filelock INFO: Lock 140403321134864 acquired on /root/.cache/huggingface/transformers/88b58bece2110e8763ccccc3f2cf3727d2759ca72eae6c299c5fb0a1b4d0b61b.c1dcf06d6a9561fe2acca9a7c9f2ec07038065d704e1984b293c6c142b53c3a8.h5.lock\n",
            "2021-04-17 10:59:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-17 10:59:07 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /overfit/peyma-ner-bert-base/d1c61bed0b0e4f6703c4df035e4c441562e478a514c4ee3e5120c8f901e837ef HTTP/1.1\" 200 649317472\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c36bc42b5a3a49b0b94c9ced194525c2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=649317472.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 10:59:26 filelock DEBUG: Attempting to release lock 140403321134864 on /root/.cache/huggingface/transformers/88b58bece2110e8763ccccc3f2cf3727d2759ca72eae6c299c5fb0a1b4d0b61b.c1dcf06d6a9561fe2acca9a7c9f2ec07038065d704e1984b293c6c142b53c3a8.h5.lock\n",
            "2021-04-17 10:59:26 filelock INFO: Lock 140403321134864 released on /root/.cache/huggingface/transformers/88b58bece2110e8763ccccc3f2cf3727d2759ca72eae6c299c5fb0a1b4d0b61b.c1dcf06d6a9561fe2acca9a7c9f2ec07038065d704e1984b293c6c142b53c3a8.h5.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at overfit/peyma-ner-bert-base were not used when initializing TFBertForTokenClassification: ['dropout_75']\n",
            "- This IS expected if you are initializing TFBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at overfit/peyma-ner-bert-base.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W1qWIXORe-ta",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f1c6eae-b71c-4ddc-8158-2c1a8a204881"
      },
      "source": [
        "tag_map = {\n",
        "  \"B_LOC\": \"B_LOC\",\n",
        "  \"B_ORG\": \"B_ORG\",\n",
        "  \"B_PER\": \"B_PER\",\n",
        "  \"B_MON\": \"B_MON\",\n",
        "  \"B_PCT\": \"B_PCT\",\n",
        "  \"B_DAT\": \"B_DAT\",\n",
        "  \"B_TIM\": \"B_TIM\",\n",
        "  \"I_LOC\": \"I_LOC\",\n",
        "  \"I_ORG\": \"I_ORG\",\n",
        "  \"I_PER\": \"I_PER\",\n",
        "  \"I_MON\": \"I_MON\",\n",
        "  \"I_PCT\": \"I_PCT\",\n",
        "  \"I_DAT\": \"I_DAT\",\n",
        "  \"I_TIM\": \"I_TIM\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts_peyma, tags_peyma, peyma_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'MON', 'PCT', 'DAT', 'TIM'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:17:04 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 1006,\n",
              "   'correct': 917,\n",
              "   'incorrect': 23,\n",
              "   'missed': 38,\n",
              "   'partial': 0,\n",
              "   'possible': 978,\n",
              "   'precision': 0.911530815109344,\n",
              "   'recall': 0.9376278118609407,\n",
              "   'spurious': 66},\n",
              "  'exact': {'actual': 1006,\n",
              "   'correct': 902,\n",
              "   'incorrect': 38,\n",
              "   'missed': 38,\n",
              "   'partial': 0,\n",
              "   'possible': 978,\n",
              "   'precision': 0.8966202783300199,\n",
              "   'recall': 0.9222903885480572,\n",
              "   'spurious': 66},\n",
              "  'partial': {'actual': 968,\n",
              "   'correct': 902,\n",
              "   'incorrect': 0,\n",
              "   'missed': 38,\n",
              "   'partial': 38,\n",
              "   'possible': 978,\n",
              "   'precision': 0.9514462809917356,\n",
              "   'recall': 0.941717791411043,\n",
              "   'spurious': 66},\n",
              "  'strict': {'actual': 1006,\n",
              "   'correct': 887,\n",
              "   'incorrect': 53,\n",
              "   'missed': 38,\n",
              "   'partial': 0,\n",
              "   'possible': 978,\n",
              "   'precision': 0.8817097415506958,\n",
              "   'recall': 0.9069529652351738,\n",
              "   'spurious': 66}},\n",
              " {'DAT': {'ent_type': {'actual': 104,\n",
              "    'correct': 92,\n",
              "    'incorrect': 1,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 98,\n",
              "    'precision': 0.8846153846153846,\n",
              "    'recall': 0.9387755102040817,\n",
              "    'spurious': 11},\n",
              "   'exact': {'actual': 104,\n",
              "    'correct': 84,\n",
              "    'incorrect': 9,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 98,\n",
              "    'precision': 0.8076923076923077,\n",
              "    'recall': 0.8571428571428571,\n",
              "    'spurious': 11},\n",
              "   'partial': {'actual': 95,\n",
              "    'correct': 84,\n",
              "    'incorrect': 0,\n",
              "    'missed': 5,\n",
              "    'partial': 9,\n",
              "    'possible': 98,\n",
              "    'precision': 0.9315789473684211,\n",
              "    'recall': 0.9030612244897959,\n",
              "    'spurious': 11},\n",
              "   'strict': {'actual': 104,\n",
              "    'correct': 83,\n",
              "    'incorrect': 10,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 98,\n",
              "    'precision': 0.7980769230769231,\n",
              "    'recall': 0.8469387755102041,\n",
              "    'spurious': 11}},\n",
              "  'LOC': {'ent_type': {'actual': 315,\n",
              "    'correct': 284,\n",
              "    'incorrect': 3,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9015873015873016,\n",
              "    'recall': 0.9281045751633987,\n",
              "    'spurious': 28},\n",
              "   'exact': {'actual': 315,\n",
              "    'correct': 278,\n",
              "    'incorrect': 9,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8825396825396825,\n",
              "    'recall': 0.9084967320261438,\n",
              "    'spurious': 28},\n",
              "   'partial': {'actual': 306,\n",
              "    'correct': 278,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 9,\n",
              "    'possible': 306,\n",
              "    'precision': 0.923202614379085,\n",
              "    'recall': 0.923202614379085,\n",
              "    'spurious': 28},\n",
              "   'strict': {'actual': 315,\n",
              "    'correct': 276,\n",
              "    'incorrect': 11,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8761904761904762,\n",
              "    'recall': 0.9019607843137255,\n",
              "    'spurious': 28}},\n",
              "  'MON': {'ent_type': {'actual': 11,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.9090909090909091,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 11,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.9090909090909091,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 11,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.9090909090909091,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 11,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.9090909090909091,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1}},\n",
              "  'ORG': {'ent_type': {'actual': 316,\n",
              "    'correct': 282,\n",
              "    'incorrect': 16,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 311,\n",
              "    'precision': 0.8924050632911392,\n",
              "    'recall': 0.9067524115755627,\n",
              "    'spurious': 18},\n",
              "   'exact': {'actual': 316,\n",
              "    'correct': 280,\n",
              "    'incorrect': 18,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 311,\n",
              "    'precision': 0.8860759493670886,\n",
              "    'recall': 0.9003215434083601,\n",
              "    'spurious': 18},\n",
              "   'partial': {'actual': 298,\n",
              "    'correct': 280,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 18,\n",
              "    'possible': 311,\n",
              "    'precision': 0.9697986577181208,\n",
              "    'recall': 0.9292604501607717,\n",
              "    'spurious': 18},\n",
              "   'strict': {'actual': 316,\n",
              "    'correct': 271,\n",
              "    'incorrect': 27,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 311,\n",
              "    'precision': 0.8575949367088608,\n",
              "    'recall': 0.8713826366559485,\n",
              "    'spurious': 18}},\n",
              "  'PCT': {'ent_type': {'actual': 18,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.9444444444444444,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 18,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.9444444444444444,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 18,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.9444444444444444,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 18,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.9444444444444444,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1}},\n",
              "  'PER': {'ent_type': {'actual': 233,\n",
              "    'correct': 224,\n",
              "    'incorrect': 2,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9613733905579399,\n",
              "    'recall': 0.986784140969163,\n",
              "    'spurious': 7},\n",
              "   'exact': {'actual': 233,\n",
              "    'correct': 224,\n",
              "    'incorrect': 2,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9613733905579399,\n",
              "    'recall': 0.986784140969163,\n",
              "    'spurious': 7},\n",
              "   'partial': {'actual': 231,\n",
              "    'correct': 224,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 2,\n",
              "    'possible': 227,\n",
              "    'precision': 0.974025974025974,\n",
              "    'recall': 0.9911894273127754,\n",
              "    'spurious': 7},\n",
              "   'strict': {'actual': 233,\n",
              "    'correct': 222,\n",
              "    'incorrect': 4,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9527896995708155,\n",
              "    'recall': 0.9779735682819384,\n",
              "    'spurious': 7}},\n",
              "  'TIM': {'ent_type': {'actual': 9,\n",
              "    'correct': 8,\n",
              "    'incorrect': 1,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 0.8888888888888888,\n",
              "    'recall': 0.8888888888888888,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 9,\n",
              "    'correct': 8,\n",
              "    'incorrect': 1,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 0.8888888888888888,\n",
              "    'recall': 0.8888888888888888,\n",
              "    'spurious': 0}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_K6WxAeYoch3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34940452-2665-4d74-9e04-7d3e1bfb2791"
      },
      "source": [
        "tag_map = {\n",
        "  \"B_LOC\": \"B_LOC\",\n",
        "  \"B_ORG\": \"B_ORG\",\n",
        "  \"B_PER\": \"B_PER\",\n",
        "  \"B_MON\": \"B_MON\",\n",
        "  \"B_PCT\": \"B_PCT\",\n",
        "  \"B_DAT\": \"B_DAT\",\n",
        "  \"B_TIM\": \"B_TIM\",\n",
        "  \"I_LOC\": \"I_LOC\",\n",
        "  \"I_ORG\": \"I_ORG\",\n",
        "  \"I_PER\": \"I_PER\",\n",
        "  \"I_MON\": \"I_MON\",\n",
        "  \"I_PCT\": \"I_PCT\",\n",
        "  \"I_DAT\": \"I_DAT\",\n",
        "  \"I_TIM\": \"I_TIM\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts_peyma, tags_peyma, peyma_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:03 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 864,\n",
              "   'correct': 790,\n",
              "   'incorrect': 21,\n",
              "   'missed': 33,\n",
              "   'partial': 0,\n",
              "   'possible': 844,\n",
              "   'precision': 0.9143518518518519,\n",
              "   'recall': 0.9360189573459715,\n",
              "   'spurious': 53},\n",
              "  'exact': {'actual': 864,\n",
              "   'correct': 782,\n",
              "   'incorrect': 29,\n",
              "   'missed': 33,\n",
              "   'partial': 0,\n",
              "   'possible': 844,\n",
              "   'precision': 0.9050925925925926,\n",
              "   'recall': 0.9265402843601895,\n",
              "   'spurious': 53},\n",
              "  'partial': {'actual': 835,\n",
              "   'correct': 782,\n",
              "   'incorrect': 0,\n",
              "   'missed': 33,\n",
              "   'partial': 29,\n",
              "   'possible': 844,\n",
              "   'precision': 0.9538922155688623,\n",
              "   'recall': 0.9437203791469194,\n",
              "   'spurious': 53},\n",
              "  'strict': {'actual': 864,\n",
              "   'correct': 769,\n",
              "   'incorrect': 42,\n",
              "   'missed': 33,\n",
              "   'partial': 0,\n",
              "   'possible': 844,\n",
              "   'precision': 0.8900462962962963,\n",
              "   'recall': 0.9111374407582938,\n",
              "   'spurious': 53}},\n",
              " {'LOC': {'ent_type': {'actual': 315,\n",
              "    'correct': 284,\n",
              "    'incorrect': 3,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9015873015873016,\n",
              "    'recall': 0.9281045751633987,\n",
              "    'spurious': 28},\n",
              "   'exact': {'actual': 315,\n",
              "    'correct': 278,\n",
              "    'incorrect': 9,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8825396825396825,\n",
              "    'recall': 0.9084967320261438,\n",
              "    'spurious': 28},\n",
              "   'partial': {'actual': 306,\n",
              "    'correct': 278,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 9,\n",
              "    'possible': 306,\n",
              "    'precision': 0.923202614379085,\n",
              "    'recall': 0.923202614379085,\n",
              "    'spurious': 28},\n",
              "   'strict': {'actual': 315,\n",
              "    'correct': 276,\n",
              "    'incorrect': 11,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8761904761904762,\n",
              "    'recall': 0.9019607843137255,\n",
              "    'spurious': 28}},\n",
              "  'ORG': {'ent_type': {'actual': 316,\n",
              "    'correct': 282,\n",
              "    'incorrect': 16,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 311,\n",
              "    'precision': 0.8924050632911392,\n",
              "    'recall': 0.9067524115755627,\n",
              "    'spurious': 18},\n",
              "   'exact': {'actual': 316,\n",
              "    'correct': 280,\n",
              "    'incorrect': 18,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 311,\n",
              "    'precision': 0.8860759493670886,\n",
              "    'recall': 0.9003215434083601,\n",
              "    'spurious': 18},\n",
              "   'partial': {'actual': 298,\n",
              "    'correct': 280,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 18,\n",
              "    'possible': 311,\n",
              "    'precision': 0.9697986577181208,\n",
              "    'recall': 0.9292604501607717,\n",
              "    'spurious': 18},\n",
              "   'strict': {'actual': 316,\n",
              "    'correct': 271,\n",
              "    'incorrect': 27,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 311,\n",
              "    'precision': 0.8575949367088608,\n",
              "    'recall': 0.8713826366559485,\n",
              "    'spurious': 18}},\n",
              "  'PER': {'ent_type': {'actual': 233,\n",
              "    'correct': 224,\n",
              "    'incorrect': 2,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9613733905579399,\n",
              "    'recall': 0.986784140969163,\n",
              "    'spurious': 7},\n",
              "   'exact': {'actual': 233,\n",
              "    'correct': 224,\n",
              "    'incorrect': 2,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9613733905579399,\n",
              "    'recall': 0.986784140969163,\n",
              "    'spurious': 7},\n",
              "   'partial': {'actual': 231,\n",
              "    'correct': 224,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 2,\n",
              "    'possible': 227,\n",
              "    'precision': 0.974025974025974,\n",
              "    'recall': 0.9911894273127754,\n",
              "    'spurious': 7},\n",
              "   'strict': {'actual': 233,\n",
              "    'correct': 222,\n",
              "    'incorrect': 4,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9527896995708155,\n",
              "    'recall': 0.9779735682819384,\n",
              "    'spurious': 7}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0AmsxelZf3O0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c139483f-1861-4112-934a-78d0acefea38"
      },
      "source": [
        "tag_map = {\n",
        "  \"B_LOC\": \"B-LOC\",\n",
        "  \"B_ORG\": \"B-ORG\",\n",
        "  \"B_PER\": \"B-PER\",\n",
        "  \"B_MON\": \"O\",\n",
        "  \"B_PCT\": \"O\",\n",
        "  \"B_DAT\": \"O\",\n",
        "  \"B_TIM\": \"O\",\n",
        "  \"I_LOC\": \"I-LOC\",\n",
        "  \"I_ORG\": \"I-ORG\",\n",
        "  \"I_PER\": \"I-PER\",\n",
        "  \"I_MON\": \"O\",\n",
        "  \"I_PCT\": \"O\",\n",
        "  \"I_DAT\": \"O\",\n",
        "  \"I_TIM\": \"O\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts, tags, peyma_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:52:40 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 628,\n",
              "   'correct': 477,\n",
              "   'incorrect': 22,\n",
              "   'missed': 95,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.7595541401273885,\n",
              "   'recall': 0.803030303030303,\n",
              "   'spurious': 129},\n",
              "  'exact': {'actual': 628,\n",
              "   'correct': 479,\n",
              "   'incorrect': 20,\n",
              "   'missed': 95,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.7627388535031847,\n",
              "   'recall': 0.8063973063973064,\n",
              "   'spurious': 129},\n",
              "  'partial': {'actual': 608,\n",
              "   'correct': 479,\n",
              "   'incorrect': 0,\n",
              "   'missed': 95,\n",
              "   'partial': 20,\n",
              "   'possible': 594,\n",
              "   'precision': 0.8042763157894737,\n",
              "   'recall': 0.8232323232323232,\n",
              "   'spurious': 129},\n",
              "  'strict': {'actual': 628,\n",
              "   'correct': 460,\n",
              "   'incorrect': 39,\n",
              "   'missed': 95,\n",
              "   'partial': 0,\n",
              "   'possible': 594,\n",
              "   'precision': 0.732484076433121,\n",
              "   'recall': 0.7744107744107744,\n",
              "   'spurious': 129}},\n",
              " {'LOC': {'ent_type': {'actual': 230,\n",
              "    'correct': 170,\n",
              "    'incorrect': 8,\n",
              "    'missed': 43,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7391304347826086,\n",
              "    'recall': 0.7692307692307693,\n",
              "    'spurious': 52},\n",
              "   'exact': {'actual': 230,\n",
              "    'correct': 171,\n",
              "    'incorrect': 7,\n",
              "    'missed': 43,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7434782608695653,\n",
              "    'recall': 0.7737556561085973,\n",
              "    'spurious': 52},\n",
              "   'partial': {'actual': 223,\n",
              "    'correct': 171,\n",
              "    'incorrect': 0,\n",
              "    'missed': 43,\n",
              "    'partial': 7,\n",
              "    'possible': 221,\n",
              "    'precision': 0.7825112107623319,\n",
              "    'recall': 0.7895927601809954,\n",
              "    'spurious': 52},\n",
              "   'strict': {'actual': 230,\n",
              "    'correct': 165,\n",
              "    'incorrect': 13,\n",
              "    'missed': 43,\n",
              "    'partial': 0,\n",
              "    'possible': 221,\n",
              "    'precision': 0.717391304347826,\n",
              "    'recall': 0.746606334841629,\n",
              "    'spurious': 52}},\n",
              "  'ORG': {'ent_type': {'actual': 149,\n",
              "    'correct': 87,\n",
              "    'incorrect': 8,\n",
              "    'missed': 34,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5838926174496645,\n",
              "    'recall': 0.6744186046511628,\n",
              "    'spurious': 54},\n",
              "   'exact': {'actual': 149,\n",
              "    'correct': 88,\n",
              "    'incorrect': 7,\n",
              "    'missed': 34,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5906040268456376,\n",
              "    'recall': 0.6821705426356589,\n",
              "    'spurious': 54},\n",
              "   'partial': {'actual': 142,\n",
              "    'correct': 88,\n",
              "    'incorrect': 0,\n",
              "    'missed': 34,\n",
              "    'partial': 7,\n",
              "    'possible': 129,\n",
              "    'precision': 0.6443661971830986,\n",
              "    'recall': 0.7093023255813954,\n",
              "    'spurious': 54},\n",
              "   'strict': {'actual': 149,\n",
              "    'correct': 80,\n",
              "    'incorrect': 15,\n",
              "    'missed': 34,\n",
              "    'partial': 0,\n",
              "    'possible': 129,\n",
              "    'precision': 0.5369127516778524,\n",
              "    'recall': 0.6201550387596899,\n",
              "    'spurious': 54}},\n",
              "  'PER': {'ent_type': {'actual': 249,\n",
              "    'correct': 220,\n",
              "    'incorrect': 6,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8835341365461847,\n",
              "    'recall': 0.9016393442622951,\n",
              "    'spurious': 23},\n",
              "   'exact': {'actual': 249,\n",
              "    'correct': 220,\n",
              "    'incorrect': 6,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8835341365461847,\n",
              "    'recall': 0.9016393442622951,\n",
              "    'spurious': 23},\n",
              "   'partial': {'actual': 243,\n",
              "    'correct': 220,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 6,\n",
              "    'possible': 244,\n",
              "    'precision': 0.9176954732510288,\n",
              "    'recall': 0.9139344262295082,\n",
              "    'spurious': 23},\n",
              "   'strict': {'actual': 249,\n",
              "    'correct': 215,\n",
              "    'incorrect': 11,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 244,\n",
              "    'precision': 0.8634538152610441,\n",
              "    'recall': 0.8811475409836066,\n",
              "    'spurious': 23}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aJPQRmbkohDn"
      },
      "source": [
        "### Pars BErt Peyma"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PNw9x3BqoirR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "5f1d669af757410fbca1d82de36ee6d5",
            "f29b1d66c94446b88161d3eaa860a7af",
            "ca06c75378974a2e96295c0138d5096f",
            "87cb6c0ff37e48b8b8ea04817a82f4a1",
            "981b5832a12142d1bc8a9fda7e01482c",
            "4c7273e6e8fc4d32ab2c8212b1b222f2",
            "f3b8ed0006df4f998d14fa9f9937e6df",
            "82e826dd3af44669b05c0554159bbdb5",
            "16bc9f6b41da497f9a8615af4f746fd8",
            "a3890ea815e640e2a2ae0fef295a5eb0",
            "d41f090bc1f344a8b2a508d2cb92f457",
            "df23a558ba51482db899018bd43270b4",
            "c9d747aeb992474992a2b32c32ea23eb",
            "0c11a461d82e472990ebba1bc9a298b5",
            "bf5fd737b3824372af891910953228dc",
            "2f083bd1d45a4edaa63628a6a8c528db",
            "9a1ee17260b4493ca12780c7580a5e8c",
            "fa0103e1b80e44b1824fc0729d32a46c",
            "6291961738354e6298fc3189fc19ed8a",
            "691fd430de6046108ac06ec1318d8632",
            "d3493eac3ff64868b49f2991cb0750c8",
            "dce1064e787140239f4988baf02c151d",
            "c35a378cf38f4e2c84b0ba44c347fe3a",
            "fab34d6581ff49fd9fbdd2cc7519a3e3",
            "e97b653a23014991a3032fa9645556b6",
            "f0bedb8936b2432da1e8608ceb4d030b",
            "d2b9a8068351447baad4040b702cd9f4",
            "6ff66087fdc047e1bcd0ca306630938c",
            "7b14b6ab59cd4e50bfa9ee040c6637cf",
            "bd3b40ee26114c739d037a42bd66a1f2",
            "7d941b34a1824f00a399ebfcff673c5b",
            "83bc89ff87884f83aa879fc717dd024e",
            "e1d7a073749d4d8abe13643e76eb10a7",
            "593fa987743142a3ba348d8f9bb1bc90",
            "f7acd42c219349cea9401edc9cf04918",
            "8d2e78f27fef4d73857be4b3547a742f",
            "08ed57cbf4824f64bcc3293373d90a66",
            "90d9f097252749c98c308dfc5c48ee72",
            "c6fa55ed77164bb490c541ed42a2429d",
            "dd125644642f4db3949075538f58f225"
          ]
        },
        "outputId": "a8f2d9de-41c4-4147-d1be-63a05eaa8cf3"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"HooshvareLab/bert-base-parsbert-peymaner-uncased\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"HooshvareLab/bert-base-parsbert-peymaner-uncased\")\n",
        "model.eval()\n",
        "parsbert_peyma = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 11:43:07 filelock DEBUG: Attempting to acquire lock 140403308553680 on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-17 11:43:07 filelock INFO: Lock 140403308553680 acquired on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/config.json HTTP/1.1\" 200 997\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5f1d669af757410fbca1d82de36ee6d5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=997.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:07 filelock DEBUG: Attempting to release lock 140403308553680 on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-17 11:43:07 filelock INFO: Lock 140403308553680 released on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-17 11:43:07 filelock DEBUG: Attempting to acquire lock 140403179728400 on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 11:43:07 filelock INFO: Lock 140403179728400 acquired on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 1215509\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "16bc9f6b41da497f9a8615af4f746fd8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:07 filelock DEBUG: Attempting to release lock 140403179728400 on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 11:43:07 filelock INFO: Lock 140403179728400 released on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-17 11:43:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-17 11:43:08 filelock DEBUG: Attempting to acquire lock 140403308553104 on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 11:43:08 filelock INFO: Lock 140403308553104 acquired on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 112\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a1ee17260b4493ca12780c7580a5e8c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:08 filelock DEBUG: Attempting to release lock 140403308553104 on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 11:43:08 filelock INFO: Lock 140403308553104 released on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 11:43:08 filelock DEBUG: Attempting to acquire lock 140403179728400 on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-17 11:43:08 filelock INFO: Lock 140403179728400 acquired on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e97b653a23014991a3032fa9645556b6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:08 filelock DEBUG: Attempting to release lock 140403179728400 on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-17 11:43:08 filelock INFO: Lock 140403179728400 released on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:08 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:09 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-17 11:43:09 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-17 11:43:09 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/pytorch_model.bin HTTP/1.1\" 302 0\n",
            "2021-04-17 11:43:09 filelock DEBUG: Attempting to acquire lock 140403179623248 on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n",
            "2021-04-17 11:43:09 filelock INFO: Lock 140403179623248 acquired on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n",
            "2021-04-17 11:43:09 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-17 11:43:09 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/e0801d8be75ae459ca661b09260e45f0c31c039022fb2062fb9d3125af07094e HTTP/1.1\" 200 651459729\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e1d7a073749d4d8abe13643e76eb10a7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=651459729.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:43:28 filelock DEBUG: Attempting to release lock 140403179623248 on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n",
            "2021-04-17 11:43:28 filelock INFO: Lock 140403179623248 released on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_h09ArLou3M",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3984d8b4-79f5-4100-efda-260c8d0a2d9d"
      },
      "source": [
        "tag_map = {\n",
        "  \"B_LOC\": \"B_LOC\",\n",
        "  \"B_ORG\": \"B_ORG\",\n",
        "  \"B_PER\": \"B_PER\",\n",
        "  \"B_MON\": \"B_MON\",\n",
        "  \"B_PCT\": \"B_PCT\",\n",
        "  \"B_DAT\": \"B_DAT\",\n",
        "  \"B_TIM\": \"B_TIM\",\n",
        "  \"I_LOC\": \"I_LOC\",\n",
        "  \"I_ORG\": \"I_ORG\",\n",
        "  \"I_PER\": \"I_PER\",\n",
        "  \"I_MON\": \"I_MON\",\n",
        "  \"I_PCT\": \"I_PCT\",\n",
        "  \"I_DAT\": \"I_DAT\",\n",
        "  \"I_TIM\": \"I_TIM\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts_peyma, tags_peyma, parsbert_peyma, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'MON', 'PCT', 'DAT', 'TIM'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:44:58 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 982,\n",
              "   'correct': 959,\n",
              "   'incorrect': 5,\n",
              "   'missed': 11,\n",
              "   'partial': 0,\n",
              "   'possible': 975,\n",
              "   'precision': 0.9765784114052953,\n",
              "   'recall': 0.9835897435897436,\n",
              "   'spurious': 18},\n",
              "  'exact': {'actual': 982,\n",
              "   'correct': 959,\n",
              "   'incorrect': 5,\n",
              "   'missed': 11,\n",
              "   'partial': 0,\n",
              "   'possible': 975,\n",
              "   'precision': 0.9765784114052953,\n",
              "   'recall': 0.9835897435897436,\n",
              "   'spurious': 18},\n",
              "  'partial': {'actual': 977,\n",
              "   'correct': 959,\n",
              "   'incorrect': 0,\n",
              "   'missed': 11,\n",
              "   'partial': 5,\n",
              "   'possible': 975,\n",
              "   'precision': 0.9841351074718526,\n",
              "   'recall': 0.9861538461538462,\n",
              "   'spurious': 18},\n",
              "  'strict': {'actual': 982,\n",
              "   'correct': 955,\n",
              "   'incorrect': 9,\n",
              "   'missed': 11,\n",
              "   'partial': 0,\n",
              "   'possible': 975,\n",
              "   'precision': 0.9725050916496945,\n",
              "   'recall': 0.9794871794871794,\n",
              "   'spurious': 18}},\n",
              " {'DAT': {'ent_type': {'actual': 96,\n",
              "    'correct': 95,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 97,\n",
              "    'precision': 0.9895833333333334,\n",
              "    'recall': 0.979381443298969,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 96,\n",
              "    'correct': 93,\n",
              "    'incorrect': 2,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 97,\n",
              "    'precision': 0.96875,\n",
              "    'recall': 0.9587628865979382,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 94,\n",
              "    'correct': 93,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 2,\n",
              "    'possible': 97,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.9690721649484536,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 96,\n",
              "    'correct': 93,\n",
              "    'incorrect': 2,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 97,\n",
              "    'precision': 0.96875,\n",
              "    'recall': 0.9587628865979382,\n",
              "    'spurious': 1}},\n",
              "  'LOC': {'ent_type': {'actual': 311,\n",
              "    'correct': 302,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9710610932475884,\n",
              "    'recall': 0.9869281045751634,\n",
              "    'spurious': 9},\n",
              "   'exact': {'actual': 311,\n",
              "    'correct': 300,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9646302250803859,\n",
              "    'recall': 0.9803921568627451,\n",
              "    'spurious': 9},\n",
              "   'partial': {'actual': 309,\n",
              "    'correct': 300,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 2,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9741100323624595,\n",
              "    'recall': 0.9836601307189542,\n",
              "    'spurious': 9},\n",
              "   'strict': {'actual': 311,\n",
              "    'correct': 300,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9646302250803859,\n",
              "    'recall': 0.9803921568627451,\n",
              "    'spurious': 9}},\n",
              "  'MON': {'ent_type': {'actual': 10,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 10,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 10,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 10,\n",
              "    'correct': 10,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0}},\n",
              "  'ORG': {'ent_type': {'actual': 311,\n",
              "    'correct': 303,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.9742765273311897,\n",
              "    'recall': 0.9805825242718447,\n",
              "    'spurious': 6},\n",
              "   'exact': {'actual': 311,\n",
              "    'correct': 304,\n",
              "    'incorrect': 1,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.977491961414791,\n",
              "    'recall': 0.9838187702265372,\n",
              "    'spurious': 6},\n",
              "   'partial': {'actual': 310,\n",
              "    'correct': 304,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 1,\n",
              "    'possible': 309,\n",
              "    'precision': 0.9822580645161291,\n",
              "    'recall': 0.9854368932038835,\n",
              "    'spurious': 6},\n",
              "   'strict': {'actual': 311,\n",
              "    'correct': 303,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.9742765273311897,\n",
              "    'recall': 0.9805825242718447,\n",
              "    'spurious': 6}},\n",
              "  'PCT': {'ent_type': {'actual': 17,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 17,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 17,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 17,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0}},\n",
              "  'PER': {'ent_type': {'actual': 228,\n",
              "    'correct': 223,\n",
              "    'incorrect': 3,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9780701754385965,\n",
              "    'recall': 0.9823788546255506,\n",
              "    'spurious': 2},\n",
              "   'exact': {'actual': 228,\n",
              "    'correct': 226,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9912280701754386,\n",
              "    'recall': 0.9955947136563876,\n",
              "    'spurious': 2},\n",
              "   'partial': {'actual': 228,\n",
              "    'correct': 226,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9912280701754386,\n",
              "    'recall': 0.9955947136563876,\n",
              "    'spurious': 2},\n",
              "   'strict': {'actual': 228,\n",
              "    'correct': 223,\n",
              "    'incorrect': 3,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9780701754385965,\n",
              "    'recall': 0.9823788546255506,\n",
              "    'spurious': 2}},\n",
              "  'TIM': {'ent_type': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Crhf0aLo7Qh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "13257dbf-9fcc-4426-fa51-14f1a56152e8"
      },
      "source": [
        "tag_map = {\n",
        "  \"B_LOC\": \"B_LOC\",\n",
        "  \"B_ORG\": \"B_ORG\",\n",
        "  \"B_PER\": \"B_PER\",\n",
        "  \"B_MON\": \"B_MON\",\n",
        "  \"B_PCT\": \"B_PCT\",\n",
        "  \"B_DAT\": \"B_DAT\",\n",
        "  \"B_TIM\": \"B_TIM\",\n",
        "  \"I_LOC\": \"I_LOC\",\n",
        "  \"I_ORG\": \"I_ORG\",\n",
        "  \"I_PER\": \"I_PER\",\n",
        "  \"I_MON\": \"I_MON\",\n",
        "  \"I_PCT\": \"I_PCT\",\n",
        "  \"I_DAT\": \"I_DAT\",\n",
        "  \"I_TIM\": \"I_TIM\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(texts_peyma, tags_peyma, parsbert_peyma, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-17 11:46:17 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 850,\n",
              "   'correct': 828,\n",
              "   'incorrect': 5,\n",
              "   'missed': 9,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.9741176470588235,\n",
              "   'recall': 0.9833729216152018,\n",
              "   'spurious': 17},\n",
              "  'exact': {'actual': 850,\n",
              "   'correct': 830,\n",
              "   'incorrect': 3,\n",
              "   'missed': 9,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.9764705882352941,\n",
              "   'recall': 0.9857482185273159,\n",
              "   'spurious': 17},\n",
              "  'partial': {'actual': 847,\n",
              "   'correct': 830,\n",
              "   'incorrect': 0,\n",
              "   'missed': 9,\n",
              "   'partial': 3,\n",
              "   'possible': 842,\n",
              "   'precision': 0.9817001180637545,\n",
              "   'recall': 0.9875296912114014,\n",
              "   'spurious': 17},\n",
              "  'strict': {'actual': 850,\n",
              "   'correct': 826,\n",
              "   'incorrect': 7,\n",
              "   'missed': 9,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.971764705882353,\n",
              "   'recall': 0.9809976247030879,\n",
              "   'spurious': 17}},\n",
              " {'LOC': {'ent_type': {'actual': 311,\n",
              "    'correct': 302,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9710610932475884,\n",
              "    'recall': 0.9869281045751634,\n",
              "    'spurious': 9},\n",
              "   'exact': {'actual': 311,\n",
              "    'correct': 300,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9646302250803859,\n",
              "    'recall': 0.9803921568627451,\n",
              "    'spurious': 9},\n",
              "   'partial': {'actual': 309,\n",
              "    'correct': 300,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 2,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9741100323624595,\n",
              "    'recall': 0.9836601307189542,\n",
              "    'spurious': 9},\n",
              "   'strict': {'actual': 311,\n",
              "    'correct': 300,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.9646302250803859,\n",
              "    'recall': 0.9803921568627451,\n",
              "    'spurious': 9}},\n",
              "  'ORG': {'ent_type': {'actual': 311,\n",
              "    'correct': 303,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.9742765273311897,\n",
              "    'recall': 0.9805825242718447,\n",
              "    'spurious': 6},\n",
              "   'exact': {'actual': 311,\n",
              "    'correct': 304,\n",
              "    'incorrect': 1,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.977491961414791,\n",
              "    'recall': 0.9838187702265372,\n",
              "    'spurious': 6},\n",
              "   'partial': {'actual': 310,\n",
              "    'correct': 304,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 1,\n",
              "    'possible': 309,\n",
              "    'precision': 0.9822580645161291,\n",
              "    'recall': 0.9854368932038835,\n",
              "    'spurious': 6},\n",
              "   'strict': {'actual': 311,\n",
              "    'correct': 303,\n",
              "    'incorrect': 2,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.9742765273311897,\n",
              "    'recall': 0.9805825242718447,\n",
              "    'spurious': 6}},\n",
              "  'PER': {'ent_type': {'actual': 228,\n",
              "    'correct': 223,\n",
              "    'incorrect': 3,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9780701754385965,\n",
              "    'recall': 0.9823788546255506,\n",
              "    'spurious': 2},\n",
              "   'exact': {'actual': 228,\n",
              "    'correct': 226,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9912280701754386,\n",
              "    'recall': 0.9955947136563876,\n",
              "    'spurious': 2},\n",
              "   'partial': {'actual': 228,\n",
              "    'correct': 226,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9912280701754386,\n",
              "    'recall': 0.9955947136563876,\n",
              "    'spurious': 2},\n",
              "   'strict': {'actual': 228,\n",
              "    'correct': 223,\n",
              "    'incorrect': 3,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.9780701754385965,\n",
              "    'recall': 0.9823788546255506,\n",
              "    'spurious': 2}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZKHtG9DRo-Np"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g0z0C1iqpY_5"
      },
      "source": [
        "### Test with Masked entities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TWoMeELH2Hmv"
      },
      "source": [
        "new_text, new_tag = [], []\n",
        "for text, tweet_tag in zip(texts, tags):\n",
        "  if len([tag for tag in tweet_tag if tag.startswith('B')]) == 1:\n",
        "    new_text.append(text)\n",
        "    new_tag.append(tweet_tag)"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kVKEQleA2lbc",
        "outputId": "b5478381-f9b7-4369-f07a-ba13c018968a"
      },
      "source": [
        "len([x for x in new_tag if 'B-POG' in x])"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8Y36qGtpcnp"
      },
      "source": [
        "masked_texts = []\n",
        "for text, tweet_tag in zip(new_text, new_tag):\n",
        "  masked_text = []\n",
        "  for word, tag in zip(text, tweet_tag):\n",
        "    if tag != 'O':\n",
        "      masked_text.append('[MASK]')\n",
        "    else:\n",
        "      masked_text.append(word)\n",
        "  masked_texts.append(masked_text)"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k1sM8siOqCOS"
      },
      "source": [
        "with open('masked_texts_test.txt', 'w') as file:\n",
        "  for text in masked_texts:\n",
        "    file.write('\\n'.join(text))\n",
        "    file.write('\\n\\n')"
      ],
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t88UUrNvrAMV",
        "outputId": "9f9c7bd4-d4ca-465a-8eaf-b5579e5fdc9b"
      },
      "source": [
        "!wget -q --show-progress https://raw.githubusercontent.com/overfit-ir/persian-twitter-ner/master/masked_texts.txt"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\rmasked_texts.txt      0%[                    ]       0  --.-KB/s               \rmasked_texts.txt    100%[===================>]  17.43K  --.-KB/s    in 0s      \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z3I7bGxc_yOY"
      },
      "source": [
        "human_tags = []\n",
        "with open('masked_texts.txt', 'r') as file:\n",
        "  for line in file.readlines():\n",
        "    if line != '\\n':\n",
        "      s = line.split('\\t')\n",
        "      if len(s) == 1:\n",
        "        human_tags.append('O')\n",
        "      else:\n",
        "        human_tags.append(s[1].replace('\\n', ''))"
      ],
      "execution_count": 86,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ASneeXmAjYJ",
        "outputId": "0db20676-7ead-4de4-86ee-15171a4291d5"
      },
      "source": [
        "len([x for tweet_tag in new_tag for x in tweet_tag])"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2075"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fBvmf5TLAOQ5",
        "outputId": "b405529a-c53c-4a7b-d176-92a0ac338246"
      },
      "source": [
        "len(human_tags)"
      ],
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2075"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n06Ie96xAth_",
        "outputId": "8259e3ec-735e-4a4f-c59e-67180044ff13"
      },
      "source": [
        "benchmark([[x for tweet_tag in new_tag for x in tweet_tag]], [human_tags], ['PER', 'LOC', 'ORG', 'NAT', 'POG'])"
      ],
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 12:45:03 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 79,\n",
              "   'correct': 65,\n",
              "   'incorrect': 12,\n",
              "   'missed': 7,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.8227848101265823,\n",
              "   'recall': 0.7738095238095238,\n",
              "   'spurious': 2},\n",
              "  'exact': {'actual': 79,\n",
              "   'correct': 77,\n",
              "   'incorrect': 0,\n",
              "   'missed': 7,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.9746835443037974,\n",
              "   'recall': 0.9166666666666666,\n",
              "   'spurious': 2},\n",
              "  'partial': {'actual': 79,\n",
              "   'correct': 77,\n",
              "   'incorrect': 0,\n",
              "   'missed': 7,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.9746835443037974,\n",
              "   'recall': 0.9166666666666666,\n",
              "   'spurious': 2},\n",
              "  'strict': {'actual': 79,\n",
              "   'correct': 65,\n",
              "   'incorrect': 12,\n",
              "   'missed': 7,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.8227848101265823,\n",
              "   'recall': 0.7738095238095238,\n",
              "   'spurious': 2}},\n",
              " {'LOC': {'ent_type': {'actual': 19,\n",
              "    'correct': 16,\n",
              "    'incorrect': 1,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.8421052631578947,\n",
              "    'recall': 0.8,\n",
              "    'spurious': 2},\n",
              "   'exact': {'actual': 19,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.8947368421052632,\n",
              "    'recall': 0.85,\n",
              "    'spurious': 2},\n",
              "   'partial': {'actual': 19,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.8947368421052632,\n",
              "    'recall': 0.85,\n",
              "    'spurious': 2},\n",
              "   'strict': {'actual': 19,\n",
              "    'correct': 16,\n",
              "    'incorrect': 1,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.8421052631578947,\n",
              "    'recall': 0.8,\n",
              "    'spurious': 2}},\n",
              "  'NAT': {'ent_type': {'actual': 6,\n",
              "    'correct': 4,\n",
              "    'incorrect': 2,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 0.6666666666666666,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 6,\n",
              "    'correct': 6,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 6,\n",
              "    'correct': 6,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 6,\n",
              "    'correct': 4,\n",
              "    'incorrect': 2,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 0.6666666666666666,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 0}},\n",
              "  'ORG': {'ent_type': {'actual': 17,\n",
              "    'correct': 12,\n",
              "    'incorrect': 5,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 0.7058823529411765,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 17,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.9444444444444444,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 17,\n",
              "    'correct': 17,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.9444444444444444,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 17,\n",
              "    'correct': 12,\n",
              "    'incorrect': 5,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 0.7058823529411765,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 0}},\n",
              "  'PER': {'ent_type': {'actual': 35,\n",
              "    'correct': 33,\n",
              "    'incorrect': 2,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 0.9428571428571428,\n",
              "    'recall': 0.868421052631579,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 35,\n",
              "    'correct': 35,\n",
              "    'incorrect': 0,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.9210526315789473,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 35,\n",
              "    'correct': 35,\n",
              "    'incorrect': 0,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.9210526315789473,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 35,\n",
              "    'correct': 33,\n",
              "    'incorrect': 2,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 0.9428571428571428,\n",
              "    'recall': 0.868421052631579,\n",
              "    'spurious': 0}},\n",
              "  'POG': {'ent_type': {'actual': 2,\n",
              "    'correct': 0,\n",
              "    'incorrect': 2,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 2,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 2,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 1.0,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 2,\n",
              "    'correct': 0,\n",
              "    'incorrect': 2,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "ddbbd5d4e7df4616ab769e4800e1dd28",
            "3b9ef28ba713432c89bc10758df9ecaa",
            "cec72dee43574c37b7c9ecce13f23414",
            "e5c73a7d0772457fa3e32132280cac28",
            "02983885fd0f4cab9b973b2c57e7bf32",
            "55a359eba2a8477d88c4abb146c49d72",
            "8a26909b53104374ba14d64f74d4e1bb",
            "90101cece0ab4af9b9dd87992564c95b",
            "21c68bd6a69c455d8dec19130ee786fa",
            "c859846604e040dc95dd3f0eba3baebc",
            "069064350b924e71a5b7ae92cdb4cf51",
            "52486fe33a0c4dc4bd1f92d19eef9ecb",
            "2e67b8a3d5ec4408ad31458e696fdb8d",
            "098c562696554826ad5e036cf6508ff0",
            "8816d9f61912431698e080fe4cedf5aa",
            "2c06935c82cb477f9e899835bd1a75af",
            "61ca72ad4cda4018b0f608beb64ce7e6",
            "e68f9b165ffd4be784e7268add1af5d8",
            "2cbb164f7bae4b7185e99100f2e35de6",
            "2a467e95c5d9478eb4d2c5d6e4a81a28",
            "2f2ead96607c4ebd8be732ff55e2a556",
            "0aadae3b6ff947548727d31420510da4",
            "5e48e974ca654aeb897bdb7906fcce5f",
            "ddc0db2b656144109b5497d0af98c42d",
            "f3836e256f3b4d5eb9eaafbe56e1edfc",
            "80e83788717d4baaabcc030c76584660",
            "390e3dacef5c488b965e2a8643af2a70",
            "0a74233ac2504b57a3bd21a5c934b9a5",
            "9f4887836ac94e96ac45a89cb60305d2",
            "b99bc5c10603483d943c9ddf9eb20d75",
            "76a5ba4de18b4ac2908501fd9a41cdd3",
            "c1b1986e8c5d4b14855bd26f45cf84a7",
            "48a90a675bff4d4c969e0a9fa5c13424",
            "bfc36645bb4b4d37817e84d5ebc631f9",
            "15def708108f431d9ccbcb1dc3103451",
            "affbf0081d0b4243be160285bf2969ed",
            "cd34da28d73c47ef88e034d47866745f",
            "7bfc840549cb43cdb1ec2c529b21697c",
            "944c5683177147bcb947fffef1c7e644",
            "e0f211affada407a8b61df6726d7025f"
          ]
        },
        "id": "fZ2nxMB4q_iI",
        "outputId": "a1e8409f-6eb3-463f-e54c-e97e7fc7de00"
      },
      "source": [
        "from transformers import TFAutoModelForTokenClassification\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "twiner_mtl = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:53 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:09:53 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:09:53 filelock DEBUG: Attempting to acquire lock 140083018445520 on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-18 11:09:53 filelock INFO: Lock 140083018445520 acquired on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-18 11:09:53 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:09:54 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 1124\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ddbbd5d4e7df4616ab769e4800e1dd28",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1124.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:54 filelock DEBUG: Attempting to release lock 140083018445520 on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-18 11:09:54 filelock INFO: Lock 140083018445520 released on /root/.cache/huggingface/transformers/6aae46c25a22a58e18b1c1b6adc8a8b0af058d1c9d46d9235c77312bfcedcb5c.24ecbabe775c7edabcc88227cb13d0c504150601e1c0dbb1dd2c7295f6766cde.lock\n",
            "2021-04-18 11:09:54 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:55 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-18 11:09:55 filelock DEBUG: Attempting to acquire lock 140083018209872 on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:09:55 filelock INFO: Lock 140083018209872 acquired on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:09:55 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:09:55 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/vocab.txt HTTP/1.1\" 200 1215509\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "21c68bd6a69c455d8dec19130ee786fa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:56 filelock DEBUG: Attempting to release lock 140083018209872 on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:09:56 filelock INFO: Lock 140083018209872 released on /root/.cache/huggingface/transformers/0e254b340265789f234ea0a71cd8228e05e94a5d19068a74ea35abe12aa012c9.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:09:56 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:57 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-18 11:09:57 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:09:57 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-18 11:09:57 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:09:58 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:09:58 filelock DEBUG: Attempting to acquire lock 140083208211408 on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:09:58 filelock INFO: Lock 140083208211408 acquired on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:09:58 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:09:59 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/special_tokens_map.json HTTP/1.1\" 200 112\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61ca72ad4cda4018b0f608beb64ce7e6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:59 filelock DEBUG: Attempting to release lock 140083208211408 on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:09:59 filelock INFO: Lock 140083208211408 released on /root/.cache/huggingface/transformers/cfb340e9b5920789a748ef17883dab7fe74644822190f9624abb224a48e96638.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:09:59 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:09:59 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:09:59 filelock DEBUG: Attempting to acquire lock 140083208504656 on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-18 11:09:59 filelock INFO: Lock 140083208504656 acquired on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-18 11:09:59 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:10:00 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/resolve/main/tokenizer_config.json HTTP/1.1\" 200 354\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f3836e256f3b4d5eb9eaafbe56e1edfc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=354.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:10:00 filelock DEBUG: Attempting to release lock 140083208504656 on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n",
            "2021-04-18 11:10:00 filelock INFO: Lock 140083208504656 released on /root/.cache/huggingface/transformers/22ab5ca7b32afe47eef18e996cf9bf7c5a87fb7315b285918612fd9c38a56ac6.f5bfb2432b46fe53cc72280487a37c915c677add081f32fdf0d70a6dca4539ac.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:10:00 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:10:01 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:10:01 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:10:01 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tf_model.h5 HTTP/1.1\" 302 0\n",
            "2021-04-18 11:10:01 filelock DEBUG: Attempting to acquire lock 140085712401680 on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n",
            "2021-04-18 11:10:01 filelock INFO: Lock 140085712401680 acquired on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n",
            "2021-04-18 11:10:01 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-18 11:10:03 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /overfit/twiner-bert-base-mtl/2d70b4698a3d6ebcf87ee24ed94b0d5ebcbd9bc7890ae991171c3f488675f128 HTTP/1.1\" 200 649311328\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "48a90a675bff4d4c969e0a9fa5c13424",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=649311328.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:10:40 filelock DEBUG: Attempting to release lock 140085712401680 on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n",
            "2021-04-18 11:10:40 filelock INFO: Lock 140085712401680 released on /root/.cache/huggingface/transformers/3ef81553e6fd3d5f3cf0bae8f0b3e6f2a9f5ce9bd2f14a52fb1d1f23baddc3ed.c2e65ea90d8672dd4b4142ea7dd4040883e024e5a78acfc46626bed3f02421cd.h5.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForTokenClassification.\n",
            "\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at overfit/twiner-bert-base-mtl.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pdnEhv_VqM_r",
        "outputId": "9fea84b1-b64d-4509-abd8-5321009fcf91"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"B-EVE\",\n",
        "  \"B-LOC\": \"B-LOC\",\n",
        "  \"B-ORG\": \"B-ORG\",\n",
        "  \"B-PER\": \"B-PER\",\n",
        "  \"B-POG\": \"B-POG\",\n",
        "  \"B-NAT\": \"B-NAT\",\n",
        "  \"I-EVE\": \"I-EVE\",\n",
        "  \"I-LOC\": \"I-LOC\",\n",
        "  \"I-ORG\": \"I-ORG\",\n",
        "  \"I-PER\": \"I-PER\",\n",
        "  \"I-POG\": \"I-POG\",\n",
        "  \"I-NAT\": \"I-NAT\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(masked_texts, new_tag, twiner_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'NAT', 'POG'])"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 12:10:02 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 17,\n",
              "   'correct': 4,\n",
              "   'incorrect': 0,\n",
              "   'missed': 80,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.23529411764705882,\n",
              "   'recall': 0.047619047619047616,\n",
              "   'spurious': 13},\n",
              "  'exact': {'actual': 17,\n",
              "   'correct': 4,\n",
              "   'incorrect': 0,\n",
              "   'missed': 80,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.23529411764705882,\n",
              "   'recall': 0.047619047619047616,\n",
              "   'spurious': 13},\n",
              "  'partial': {'actual': 17,\n",
              "   'correct': 4,\n",
              "   'incorrect': 0,\n",
              "   'missed': 80,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.23529411764705882,\n",
              "   'recall': 0.047619047619047616,\n",
              "   'spurious': 13},\n",
              "  'strict': {'actual': 17,\n",
              "   'correct': 4,\n",
              "   'incorrect': 0,\n",
              "   'missed': 80,\n",
              "   'partial': 0,\n",
              "   'possible': 84,\n",
              "   'precision': 0.23529411764705882,\n",
              "   'recall': 0.047619047619047616,\n",
              "   'spurious': 13}},\n",
              " {'LOC': {'ent_type': {'actual': 5,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.05,\n",
              "    'spurious': 4},\n",
              "   'exact': {'actual': 5,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.05,\n",
              "    'spurious': 4},\n",
              "   'partial': {'actual': 5,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.05,\n",
              "    'spurious': 4},\n",
              "   'strict': {'actual': 5,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 20,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.05,\n",
              "    'spurious': 4}},\n",
              "  'NAT': {'ent_type': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 6,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3},\n",
              "   'exact': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 6,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3},\n",
              "   'partial': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 6,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3},\n",
              "   'strict': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 6,\n",
              "    'partial': 0,\n",
              "    'possible': 6,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3}},\n",
              "  'ORG': {'ent_type': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3},\n",
              "   'exact': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3},\n",
              "   'partial': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3},\n",
              "   'strict': {'actual': 3,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 18,\n",
              "    'partial': 0,\n",
              "    'possible': 18,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 3}},\n",
              "  'PER': {'ent_type': {'actual': 6,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07894736842105263,\n",
              "    'spurious': 3},\n",
              "   'exact': {'actual': 6,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07894736842105263,\n",
              "    'spurious': 3},\n",
              "   'partial': {'actual': 6,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07894736842105263,\n",
              "    'spurious': 3},\n",
              "   'strict': {'actual': 6,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 35,\n",
              "    'partial': 0,\n",
              "    'possible': 38,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.07894736842105263,\n",
              "    'spurious': 3}},\n",
              "  'POG': {'ent_type': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 2,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bzKRWP55s73m"
      },
      "source": [
        "masked_texts_peyma = []\n",
        "for text, tweet_tag in zip(texts_peyma, tags_peyma):\n",
        "  masked_text = []\n",
        "  for word, tag in zip(text, tweet_tag):\n",
        "    if tag != 'O':\n",
        "      masked_text.append('[MASK]')\n",
        "    else:\n",
        "      masked_text.append(word)\n",
        "  masked_texts_peyma.append(masked_text)"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "ef91b2f52698484fb89650b83333d5a7",
            "b3988b24758d4d40b03567f1b5f24e7b",
            "67c21a70804543a7850ccf66b6f80129",
            "4b736c7fd39c45339d850b751f053031",
            "75e8389e698342efb0dcc64820d78f2b",
            "61112889ca0542839573d5f4c565bdba",
            "e893da8bd2a143dabf8a03ae9eae345a",
            "4aa624c1ecbe4f1584a3d6ac0ecdc023",
            "66befa77c3944d4e9339b713bf6ecd3d",
            "81c0454bfc024f2ab6d3de09f9d9ef1b",
            "4e1e6ead3ecd447e9fe1043f987667f8",
            "54161db780ea420e8232d44200c0153a",
            "512627527a7e49cf8b1105eef1312437",
            "a7e787e127ea4291988595f4507a1afe",
            "fa8302519e0e422aaa0bc9d983357d9c",
            "cc03250ff47246778f3476e0e636914b",
            "cb0b7472c01e46f990ff056fbd7ddd22",
            "6f8dc8d1e0f9473e867504459144ebef",
            "24ab94ff8dfa4e79baf4ccc108600607",
            "e735ba01a0ac4e84b9ac2004e18f6e8e",
            "8df4818bf8cb45bb945614f900ef1a3e",
            "0ac453887ca5476c9c0a2054f0fb47e0",
            "52f2edeea77f4c1aacd17e525e8c7767",
            "1b3aef3589c745118ee14aba4ac08e7d",
            "93bee8a4880a4b27b670cad320556b02",
            "5b99105196dc4116b72823d8d9566bd0",
            "e97e1694f5484ef88c02dc60d188e30b",
            "bae3f74bf4884321983bee5955ccf1f5",
            "f23be764e14d482b97cf2b8e3046662e",
            "a015845b9c8a443abcfcf243959080f8",
            "f7b988af55664744b7dc38fe683ed8ab",
            "2271ba12f76b4c9e91684e135a2d1d72",
            "2d5bef9964ed47708973eb6e636e0c0a",
            "291888c597f84cd78874bbe324231b93",
            "337d0777f5e84d5e9af78e5bdd714a85",
            "b10f40a1bafb4e6e816cc8ba224b5b4b",
            "b6f9643c72364197ba8cb45dab2a8619",
            "b0f9f269432c4a3983ff1d4d267eb4a1",
            "1c5586da885b42e9b15021bf547d5446",
            "6ce6008cefdb404b8aa66d5220a8e6fd"
          ]
        },
        "id": "JVwXBXR6rQP7",
        "outputId": "645a5d1e-cb8c-4f81-b8e5-6e106cef0283"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"HooshvareLab/bert-base-parsbert-peymaner-uncased\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"HooshvareLab/bert-base-parsbert-peymaner-uncased\")\n",
        "model.eval()\n",
        "parsbert_peyma = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:06 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:23:06 filelock DEBUG: Attempting to acquire lock 140082938759696 on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-18 11:23:06 filelock INFO: Lock 140082938759696 acquired on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-18 11:23:06 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:07 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/config.json HTTP/1.1\" 200 997\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ef91b2f52698484fb89650b83333d5a7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=997.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:07 filelock DEBUG: Attempting to release lock 140082938759696 on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-18 11:23:07 filelock INFO: Lock 140082938759696 released on /root/.cache/huggingface/transformers/63b8a6a3548fd09df2b1c8b59dafbee427c1ec85c7a2a6e896454f11611e7f04.c4ce79e951e964f7837f2d8e269cd9a0bbcee6b5be0aeb65e8fb944d5184af3c.lock\n",
            "2021-04-18 11:23:07 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:08 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-18 11:23:08 filelock DEBUG: Attempting to acquire lock 140082937258192 on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:23:08 filelock INFO: Lock 140082937258192 acquired on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:23:08 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:08 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 1215509\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "66befa77c3944d4e9339b713bf6ecd3d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1215509.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:09 filelock DEBUG: Attempting to release lock 140082937258192 on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:23:09 filelock INFO: Lock 140082937258192 released on /root/.cache/huggingface/transformers/a636f20d387de26da664850322ecd2eb83581e329c3f6e1f6efced8d86a6d613.6699f2ee4745b6531f79b9781879071b6ace2d2768df83889391421fb44d4474.lock\n",
            "2021-04-18 11:23:09 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:10 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-18 11:23:10 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:10 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-18 11:23:10 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:11 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:23:11 filelock DEBUG: Attempting to acquire lock 140082936942864 on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:23:11 filelock INFO: Lock 140082936942864 acquired on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:23:11 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 112\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cb0b7472c01e46f990ff056fbd7ddd22",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:12 filelock DEBUG: Attempting to release lock 140082936942864 on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:23:12 filelock INFO: Lock 140082936942864 released on /root/.cache/huggingface/transformers/0ec614ae9c28937555af1539fc9e2320ebe7a5e9aecf6b712ff2bea88b62e4b8.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d.lock\n",
            "2021-04-18 11:23:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:12 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:23:12 filelock DEBUG: Attempting to acquire lock 140082938694544 on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-18 11:23:12 filelock INFO: Lock 140082938694544 acquired on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-18 11:23:12 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:13 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93bee8a4880a4b27b670cad320556b02",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:13 filelock DEBUG: Attempting to release lock 140082938694544 on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n",
            "2021-04-18 11:23:13 filelock INFO: Lock 140082938694544 released on /root/.cache/huggingface/transformers/1ad82da13039dcf9acaf9f82ffeb7366accb0f33f64b4f697a5716a69eee8285.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:23:13 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:14 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:23:14 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:23:14 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /HooshvareLab/bert-base-parsbert-peymaner-uncased/resolve/main/pytorch_model.bin HTTP/1.1\" 302 0\n",
            "2021-04-18 11:23:15 filelock DEBUG: Attempting to acquire lock 140082939016400 on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n",
            "2021-04-18 11:23:15 filelock INFO: Lock 140082939016400 acquired on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n",
            "2021-04-18 11:23:15 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): cdn-lfs.huggingface.co:443\n",
            "2021-04-18 11:23:16 urllib3.connectionpool DEBUG: https://cdn-lfs.huggingface.co:443 \"GET /HooshvareLab/bert-base-parsbert-peymaner-uncased/e0801d8be75ae459ca661b09260e45f0c31c039022fb2062fb9d3125af07094e HTTP/1.1\" 200 651459729\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2d5bef9964ed47708973eb6e636e0c0a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=651459729.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:24:10 filelock DEBUG: Attempting to release lock 140082939016400 on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n",
            "2021-04-18 11:24:10 filelock INFO: Lock 140082939016400 released on /root/.cache/huggingface/transformers/070d60b171933e7e2340a97a691b9aa5eebedc5262212f6fdd00ab95ed762986.e4ae6c5a02a126ab0d1c8352dfd857fbab8490a9460b3cc1d92227127dc21742.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GV0vJl2SuEYy",
        "outputId": "afbf3132-78d8-48dd-ff35-44844fd142bb"
      },
      "source": [
        "tag_map = {\n",
        "  \"B_LOC\": \"B_LOC\",\n",
        "  \"B_ORG\": \"B_ORG\",\n",
        "  \"B_PER\": \"B_PER\",\n",
        "  \"B_MON\": \"B_MON\",\n",
        "  \"B_PCT\": \"B_PCT\",\n",
        "  \"B_DAT\": \"B_DAT\",\n",
        "  \"B_TIM\": \"B_TIM\",\n",
        "  \"I_LOC\": \"I_LOC\",\n",
        "  \"I_ORG\": \"I_ORG\",\n",
        "  \"I_PER\": \"I_PER\",\n",
        "  \"I_MON\": \"I_MON\",\n",
        "  \"I_PCT\": \"I_PCT\",\n",
        "  \"I_DAT\": \"I_DAT\",\n",
        "  \"I_TIM\": \"I_TIM\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(masked_texts_peyma, tags_peyma, parsbert_peyma, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'MON', 'PCT', 'DAT', 'TIM'])"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:29:31 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 198,\n",
              "   'correct': 102,\n",
              "   'incorrect': 26,\n",
              "   'missed': 847,\n",
              "   'partial': 0,\n",
              "   'possible': 975,\n",
              "   'precision': 0.5151515151515151,\n",
              "   'recall': 0.10461538461538461,\n",
              "   'spurious': 70},\n",
              "  'exact': {'actual': 198,\n",
              "   'correct': 92,\n",
              "   'incorrect': 36,\n",
              "   'missed': 847,\n",
              "   'partial': 0,\n",
              "   'possible': 975,\n",
              "   'precision': 0.46464646464646464,\n",
              "   'recall': 0.09435897435897436,\n",
              "   'spurious': 70},\n",
              "  'partial': {'actual': 162,\n",
              "   'correct': 92,\n",
              "   'incorrect': 0,\n",
              "   'missed': 847,\n",
              "   'partial': 36,\n",
              "   'possible': 975,\n",
              "   'precision': 0.6790123456790124,\n",
              "   'recall': 0.11282051282051282,\n",
              "   'spurious': 70},\n",
              "  'strict': {'actual': 198,\n",
              "   'correct': 72,\n",
              "   'incorrect': 56,\n",
              "   'missed': 847,\n",
              "   'partial': 0,\n",
              "   'possible': 975,\n",
              "   'precision': 0.36363636363636365,\n",
              "   'recall': 0.07384615384615385,\n",
              "   'spurious': 70}},\n",
              " {'DAT': {'ent_type': {'actual': 62,\n",
              "    'correct': 26,\n",
              "    'incorrect': 0,\n",
              "    'missed': 71,\n",
              "    'partial': 0,\n",
              "    'possible': 97,\n",
              "    'precision': 0.41935483870967744,\n",
              "    'recall': 0.26804123711340205,\n",
              "    'spurious': 36},\n",
              "   'exact': {'actual': 62,\n",
              "    'correct': 21,\n",
              "    'incorrect': 5,\n",
              "    'missed': 71,\n",
              "    'partial': 0,\n",
              "    'possible': 97,\n",
              "    'precision': 0.3387096774193548,\n",
              "    'recall': 0.21649484536082475,\n",
              "    'spurious': 36},\n",
              "   'partial': {'actual': 57,\n",
              "    'correct': 21,\n",
              "    'incorrect': 0,\n",
              "    'missed': 71,\n",
              "    'partial': 5,\n",
              "    'possible': 97,\n",
              "    'precision': 0.41228070175438597,\n",
              "    'recall': 0.2422680412371134,\n",
              "    'spurious': 36},\n",
              "   'strict': {'actual': 62,\n",
              "    'correct': 21,\n",
              "    'incorrect': 5,\n",
              "    'missed': 71,\n",
              "    'partial': 0,\n",
              "    'possible': 97,\n",
              "    'precision': 0.3387096774193548,\n",
              "    'recall': 0.21649484536082475,\n",
              "    'spurious': 36}},\n",
              "  'LOC': {'ent_type': {'actual': 25,\n",
              "    'correct': 3,\n",
              "    'incorrect': 18,\n",
              "    'missed': 285,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.12,\n",
              "    'recall': 0.00980392156862745,\n",
              "    'spurious': 4},\n",
              "   'exact': {'actual': 25,\n",
              "    'correct': 18,\n",
              "    'incorrect': 3,\n",
              "    'missed': 285,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.72,\n",
              "    'recall': 0.058823529411764705,\n",
              "    'spurious': 4},\n",
              "   'partial': {'actual': 22,\n",
              "    'correct': 18,\n",
              "    'incorrect': 0,\n",
              "    'missed': 285,\n",
              "    'partial': 3,\n",
              "    'possible': 306,\n",
              "    'precision': 0.8863636363636364,\n",
              "    'recall': 0.06372549019607843,\n",
              "    'spurious': 4},\n",
              "   'strict': {'actual': 25,\n",
              "    'correct': 2,\n",
              "    'incorrect': 19,\n",
              "    'missed': 285,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.08,\n",
              "    'recall': 0.006535947712418301,\n",
              "    'spurious': 4}},\n",
              "  'MON': {'ent_type': {'actual': 6,\n",
              "    'correct': 1,\n",
              "    'incorrect': 4,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.16666666666666666,\n",
              "    'recall': 0.1,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 6,\n",
              "    'correct': 2,\n",
              "    'incorrect': 3,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.3333333333333333,\n",
              "    'recall': 0.2,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 3,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 5,\n",
              "    'partial': 3,\n",
              "    'possible': 10,\n",
              "    'precision': 1.1666666666666667,\n",
              "    'recall': 0.35,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 6,\n",
              "    'correct': 0,\n",
              "    'incorrect': 5,\n",
              "    'missed': 5,\n",
              "    'partial': 0,\n",
              "    'possible': 10,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 1}},\n",
              "  'ORG': {'ent_type': {'actual': 87,\n",
              "    'correct': 67,\n",
              "    'incorrect': 1,\n",
              "    'missed': 241,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.7701149425287356,\n",
              "    'recall': 0.2168284789644013,\n",
              "    'spurious': 19},\n",
              "   'exact': {'actual': 87,\n",
              "    'correct': 44,\n",
              "    'incorrect': 24,\n",
              "    'missed': 241,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.5057471264367817,\n",
              "    'recall': 0.1423948220064725,\n",
              "    'spurious': 19},\n",
              "   'partial': {'actual': 63,\n",
              "    'correct': 44,\n",
              "    'incorrect': 0,\n",
              "    'missed': 241,\n",
              "    'partial': 24,\n",
              "    'possible': 309,\n",
              "    'precision': 0.8888888888888888,\n",
              "    'recall': 0.18122977346278318,\n",
              "    'spurious': 19},\n",
              "   'strict': {'actual': 87,\n",
              "    'correct': 44,\n",
              "    'incorrect': 24,\n",
              "    'missed': 241,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.5057471264367817,\n",
              "    'recall': 0.1423948220064725,\n",
              "    'spurious': 19}},\n",
              "  'PCT': {'ent_type': {'actual': 8,\n",
              "    'correct': 3,\n",
              "    'incorrect': 1,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.375,\n",
              "    'recall': 0.17647058823529413,\n",
              "    'spurious': 4},\n",
              "   'exact': {'actual': 8,\n",
              "    'correct': 3,\n",
              "    'incorrect': 1,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.375,\n",
              "    'recall': 0.17647058823529413,\n",
              "    'spurious': 4},\n",
              "   'partial': {'actual': 7,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 13,\n",
              "    'partial': 1,\n",
              "    'possible': 17,\n",
              "    'precision': 0.5,\n",
              "    'recall': 0.20588235294117646,\n",
              "    'spurious': 4},\n",
              "   'strict': {'actual': 8,\n",
              "    'correct': 3,\n",
              "    'incorrect': 1,\n",
              "    'missed': 13,\n",
              "    'partial': 0,\n",
              "    'possible': 17,\n",
              "    'precision': 0.375,\n",
              "    'recall': 0.17647058823529413,\n",
              "    'spurious': 4}},\n",
              "  'PER': {'ent_type': {'actual': 10,\n",
              "    'correct': 2,\n",
              "    'incorrect': 2,\n",
              "    'missed': 223,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.00881057268722467,\n",
              "    'spurious': 6},\n",
              "   'exact': {'actual': 10,\n",
              "    'correct': 4,\n",
              "    'incorrect': 0,\n",
              "    'missed': 223,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.01762114537444934,\n",
              "    'spurious': 6},\n",
              "   'partial': {'actual': 10,\n",
              "    'correct': 4,\n",
              "    'incorrect': 0,\n",
              "    'missed': 223,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.4,\n",
              "    'recall': 0.01762114537444934,\n",
              "    'spurious': 6},\n",
              "   'strict': {'actual': 10,\n",
              "    'correct': 2,\n",
              "    'incorrect': 2,\n",
              "    'missed': 223,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.00881057268722467,\n",
              "    'spurious': 6}},\n",
              "  'TIM': {'ent_type': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 9,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 9,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 9,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 0,\n",
              "    'correct': 0,\n",
              "    'incorrect': 0,\n",
              "    'missed': 9,\n",
              "    'partial': 0,\n",
              "    'possible': 9,\n",
              "    'precision': 0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 0}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hRF0tQGDvCE3",
        "outputId": "f43f35d5-b800-4d29-8bd4-3e8818ea7a22"
      },
      "source": [
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:27:24 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 116,\n",
              "   'correct': 78,\n",
              "   'incorrect': 10,\n",
              "   'missed': 754,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.6724137931034483,\n",
              "   'recall': 0.09263657957244656,\n",
              "   'spurious': 28},\n",
              "  'exact': {'actual': 116,\n",
              "   'correct': 87,\n",
              "   'incorrect': 1,\n",
              "   'missed': 754,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.75,\n",
              "   'recall': 0.10332541567695962,\n",
              "   'spurious': 28},\n",
              "  'partial': {'actual': 115,\n",
              "   'correct': 87,\n",
              "   'incorrect': 0,\n",
              "   'missed': 754,\n",
              "   'partial': 1,\n",
              "   'possible': 842,\n",
              "   'precision': 0.7608695652173914,\n",
              "   'recall': 0.10391923990498812,\n",
              "   'spurious': 28},\n",
              "  'strict': {'actual': 116,\n",
              "   'correct': 77,\n",
              "   'incorrect': 11,\n",
              "   'missed': 754,\n",
              "   'partial': 0,\n",
              "   'possible': 842,\n",
              "   'precision': 0.6637931034482759,\n",
              "   'recall': 0.09144893111638955,\n",
              "   'spurious': 28}},\n",
              " {'LOC': {'ent_type': {'actual': 9,\n",
              "    'correct': 0,\n",
              "    'incorrect': 7,\n",
              "    'missed': 299,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 2},\n",
              "   'exact': {'actual': 9,\n",
              "    'correct': 7,\n",
              "    'incorrect': 0,\n",
              "    'missed': 299,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.7777777777777778,\n",
              "    'recall': 0.02287581699346405,\n",
              "    'spurious': 2},\n",
              "   'partial': {'actual': 9,\n",
              "    'correct': 7,\n",
              "    'incorrect': 0,\n",
              "    'missed': 299,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.7777777777777778,\n",
              "    'recall': 0.02287581699346405,\n",
              "    'spurious': 2},\n",
              "   'strict': {'actual': 9,\n",
              "    'correct': 0,\n",
              "    'incorrect': 7,\n",
              "    'missed': 299,\n",
              "    'partial': 0,\n",
              "    'possible': 306,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 2}},\n",
              "  'ORG': {'ent_type': {'actual': 54,\n",
              "    'correct': 38,\n",
              "    'incorrect': 1,\n",
              "    'missed': 270,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.7037037037037037,\n",
              "    'recall': 0.12297734627831715,\n",
              "    'spurious': 15},\n",
              "   'exact': {'actual': 54,\n",
              "    'correct': 38,\n",
              "    'incorrect': 1,\n",
              "    'missed': 270,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.7037037037037037,\n",
              "    'recall': 0.12297734627831715,\n",
              "    'spurious': 15},\n",
              "   'partial': {'actual': 53,\n",
              "    'correct': 38,\n",
              "    'incorrect': 0,\n",
              "    'missed': 270,\n",
              "    'partial': 1,\n",
              "    'possible': 309,\n",
              "    'precision': 0.7264150943396226,\n",
              "    'recall': 0.12459546925566344,\n",
              "    'spurious': 15},\n",
              "   'strict': {'actual': 54,\n",
              "    'correct': 37,\n",
              "    'incorrect': 2,\n",
              "    'missed': 270,\n",
              "    'partial': 0,\n",
              "    'possible': 309,\n",
              "    'precision': 0.6851851851851852,\n",
              "    'recall': 0.11974110032362459,\n",
              "    'spurious': 15}},\n",
              "  'PER': {'ent_type': {'actual': 53,\n",
              "    'correct': 40,\n",
              "    'incorrect': 2,\n",
              "    'missed': 185,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.7547169811320755,\n",
              "    'recall': 0.1762114537444934,\n",
              "    'spurious': 11},\n",
              "   'exact': {'actual': 53,\n",
              "    'correct': 42,\n",
              "    'incorrect': 0,\n",
              "    'missed': 185,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.7924528301886793,\n",
              "    'recall': 0.18502202643171806,\n",
              "    'spurious': 11},\n",
              "   'partial': {'actual': 53,\n",
              "    'correct': 42,\n",
              "    'incorrect': 0,\n",
              "    'missed': 185,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.7924528301886793,\n",
              "    'recall': 0.18502202643171806,\n",
              "    'spurious': 11},\n",
              "   'strict': {'actual': 53,\n",
              "    'correct': 40,\n",
              "    'incorrect': 2,\n",
              "    'missed': 185,\n",
              "    'partial': 0,\n",
              "    'possible': 227,\n",
              "    'precision': 0.7547169811320755,\n",
              "    'recall': 0.1762114537444934,\n",
              "    'spurious': 11}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_WCwqv15uaUB"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NC6xhzL6ww6p"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YMsezdYHwxn4"
      },
      "source": [
        "### TEst with word only"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8R4uz8PVwzmv"
      },
      "source": [
        "new_texts = []\n",
        "new_tags = []\n",
        "for text, tweet_tag in zip(texts, tags):\n",
        "  new_text = []\n",
        "  new_tag = []\n",
        "  flag = False\n",
        "  for word, tag in zip(text, tweet_tag):\n",
        "    if tag.startswith('B') and flag == False:\n",
        "      flag = True\n",
        "      new_text.append(word)\n",
        "      new_tag.append(tag)\n",
        "    elif tag.startswith('B') and flag == True:\n",
        "      new_texts.append(new_text)\n",
        "      new_tags.append(new_tag)\n",
        "      flag = True\n",
        "      new_text = []\n",
        "      new_tag = []\n",
        "      new_text.append(word)\n",
        "      new_tag.append(tag)\n",
        "    elif tag.startswith('I'):\n",
        "      new_text.append(word)\n",
        "      new_tag.append(tag)\n",
        "    elif flag:\n",
        "      new_texts.append(new_text)\n",
        "      new_tags.append(new_tag)\n",
        "      flag = False\n",
        "      new_text = []\n",
        "      new_tag = []"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QV6Bbs81xxA4",
        "outputId": "98e669e7-841e-418a-afbe-a6d95ff4ac46"
      },
      "source": [
        "new_texts[5], new_tags[5]"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['جنگ', 'دوم', 'جهانی'], ['B-EVE', 'I-EVE', 'I-EVE'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rFLpuilAzEKN",
        "outputId": "00825b21-8a65-43b4-f441-bb19e2043c52"
      },
      "source": [
        "from transformers import TFAutoModelForTokenClassification\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "twiner_mtl = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 11:49:25 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:26 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:49:26 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:26 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-18 11:49:26 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:27 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-18 11:49:27 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:28 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-18 11:49:28 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:28 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:49:28 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:29 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:49:29 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:30 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 11:49:30 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 11:49:30 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tf_model.h5 HTTP/1.1\" 302 0\n",
            "Some layers from the model checkpoint at overfit/twiner-bert-base-mtl were not used when initializing TFBertForTokenClassification: ['dropout_37']\n",
            "- This IS expected if you are initializing TFBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at overfit/twiner-bert-base-mtl.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z4WrnFUx0F-J",
        "outputId": "45bcdbfd-d4e4-4484-f838-27c82ba02983"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"B-EVE\",\n",
        "  \"B-LOC\": \"B-LOC\",\n",
        "  \"B-ORG\": \"B-ORG\",\n",
        "  \"B-PER\": \"B-PER\",\n",
        "  \"B-POG\": \"B-POG\",\n",
        "  \"B-NAT\": \"B-NAT\",\n",
        "  \"I-EVE\": \"I-EVE\",\n",
        "  \"I-LOC\": \"I-LOC\",\n",
        "  \"I-ORG\": \"I-ORG\",\n",
        "  \"I-PER\": \"I-PER\",\n",
        "  \"I-POG\": \"I-POG\",\n",
        "  \"I-NAT\": \"I-NAT\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(new_texts, new_tags, twiner_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG', 'EVE', 'POG', 'NAT'])"
      ],
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "130\n",
            "140\n",
            "150\n",
            "160\n",
            "170\n",
            "180\n",
            "190\n",
            "200\n",
            "210\n",
            "220\n",
            "230\n",
            "240\n",
            "250\n",
            "260\n",
            "270\n",
            "280\n",
            "290\n",
            "300\n",
            "310\n",
            "320\n",
            "330\n",
            "340\n",
            "350\n",
            "360\n",
            "370\n",
            "380\n",
            "390\n",
            "400\n",
            "410\n",
            "420\n",
            "430\n",
            "440\n",
            "450\n",
            "460\n",
            "470\n",
            "480\n",
            "490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 13:26:00 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 460,\n",
              "   'correct': 404,\n",
              "   'incorrect': 26,\n",
              "   'missed': 70,\n",
              "   'partial': 0,\n",
              "   'possible': 500,\n",
              "   'precision': 0.8782608695652174,\n",
              "   'recall': 0.808,\n",
              "   'spurious': 30},\n",
              "  'exact': {'actual': 460,\n",
              "   'correct': 423,\n",
              "   'incorrect': 7,\n",
              "   'missed': 70,\n",
              "   'partial': 0,\n",
              "   'possible': 500,\n",
              "   'precision': 0.9195652173913044,\n",
              "   'recall': 0.846,\n",
              "   'spurious': 30},\n",
              "  'partial': {'actual': 453,\n",
              "   'correct': 423,\n",
              "   'incorrect': 0,\n",
              "   'missed': 70,\n",
              "   'partial': 7,\n",
              "   'possible': 500,\n",
              "   'precision': 0.9415011037527594,\n",
              "   'recall': 0.853,\n",
              "   'spurious': 30},\n",
              "  'strict': {'actual': 460,\n",
              "   'correct': 397,\n",
              "   'incorrect': 33,\n",
              "   'missed': 70,\n",
              "   'partial': 0,\n",
              "   'possible': 500,\n",
              "   'precision': 0.8630434782608696,\n",
              "   'recall': 0.794,\n",
              "   'spurious': 30}},\n",
              " {'EVE': {'ent_type': {'actual': 9,\n",
              "    'correct': 9,\n",
              "    'incorrect': 0,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 12,\n",
              "    'precision': 1.0,\n",
              "    'recall': 0.75,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 9,\n",
              "    'correct': 7,\n",
              "    'incorrect': 2,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 12,\n",
              "    'precision': 0.7777777777777778,\n",
              "    'recall': 0.5833333333333334,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 7,\n",
              "    'correct': 7,\n",
              "    'incorrect': 0,\n",
              "    'missed': 3,\n",
              "    'partial': 2,\n",
              "    'possible': 12,\n",
              "    'precision': 1.1428571428571428,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 9,\n",
              "    'correct': 7,\n",
              "    'incorrect': 2,\n",
              "    'missed': 3,\n",
              "    'partial': 0,\n",
              "    'possible': 12,\n",
              "    'precision': 0.7777777777777778,\n",
              "    'recall': 0.5833333333333334,\n",
              "    'spurious': 0}},\n",
              "  'LOC': {'ent_type': {'actual': 166,\n",
              "    'correct': 147,\n",
              "    'incorrect': 7,\n",
              "    'missed': 20,\n",
              "    'partial': 0,\n",
              "    'possible': 174,\n",
              "    'precision': 0.8855421686746988,\n",
              "    'recall': 0.8448275862068966,\n",
              "    'spurious': 12},\n",
              "   'exact': {'actual': 166,\n",
              "    'correct': 154,\n",
              "    'incorrect': 0,\n",
              "    'missed': 20,\n",
              "    'partial': 0,\n",
              "    'possible': 174,\n",
              "    'precision': 0.927710843373494,\n",
              "    'recall': 0.8850574712643678,\n",
              "    'spurious': 12},\n",
              "   'partial': {'actual': 166,\n",
              "    'correct': 154,\n",
              "    'incorrect': 0,\n",
              "    'missed': 20,\n",
              "    'partial': 0,\n",
              "    'possible': 174,\n",
              "    'precision': 0.927710843373494,\n",
              "    'recall': 0.8850574712643678,\n",
              "    'spurious': 12},\n",
              "   'strict': {'actual': 166,\n",
              "    'correct': 147,\n",
              "    'incorrect': 7,\n",
              "    'missed': 20,\n",
              "    'partial': 0,\n",
              "    'possible': 174,\n",
              "    'precision': 0.8855421686746988,\n",
              "    'recall': 0.8448275862068966,\n",
              "    'spurious': 12}},\n",
              "  'NAT': {'ent_type': {'actual': 24,\n",
              "    'correct': 19,\n",
              "    'incorrect': 3,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 23,\n",
              "    'precision': 0.7916666666666666,\n",
              "    'recall': 0.8260869565217391,\n",
              "    'spurious': 2},\n",
              "   'exact': {'actual': 24,\n",
              "    'correct': 22,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 23,\n",
              "    'precision': 0.9166666666666666,\n",
              "    'recall': 0.9565217391304348,\n",
              "    'spurious': 2},\n",
              "   'partial': {'actual': 24,\n",
              "    'correct': 22,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 23,\n",
              "    'precision': 0.9166666666666666,\n",
              "    'recall': 0.9565217391304348,\n",
              "    'spurious': 2},\n",
              "   'strict': {'actual': 24,\n",
              "    'correct': 19,\n",
              "    'incorrect': 3,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 23,\n",
              "    'precision': 0.7916666666666666,\n",
              "    'recall': 0.8260869565217391,\n",
              "    'spurious': 2}},\n",
              "  'ORG': {'ent_type': {'actual': 97,\n",
              "    'correct': 80,\n",
              "    'incorrect': 10,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 109,\n",
              "    'precision': 0.8247422680412371,\n",
              "    'recall': 0.7339449541284404,\n",
              "    'spurious': 7},\n",
              "   'exact': {'actual': 97,\n",
              "    'correct': 89,\n",
              "    'incorrect': 1,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 109,\n",
              "    'precision': 0.9175257731958762,\n",
              "    'recall': 0.8165137614678899,\n",
              "    'spurious': 7},\n",
              "   'partial': {'actual': 96,\n",
              "    'correct': 89,\n",
              "    'incorrect': 0,\n",
              "    'missed': 19,\n",
              "    'partial': 1,\n",
              "    'possible': 109,\n",
              "    'precision': 0.9322916666666666,\n",
              "    'recall': 0.8211009174311926,\n",
              "    'spurious': 7},\n",
              "   'strict': {'actual': 97,\n",
              "    'correct': 79,\n",
              "    'incorrect': 11,\n",
              "    'missed': 19,\n",
              "    'partial': 0,\n",
              "    'possible': 109,\n",
              "    'precision': 0.8144329896907216,\n",
              "    'recall': 0.7247706422018348,\n",
              "    'spurious': 7}},\n",
              "  'PER': {'ent_type': {'actual': 152,\n",
              "    'correct': 141,\n",
              "    'incorrect': 2,\n",
              "    'missed': 23,\n",
              "    'partial': 0,\n",
              "    'possible': 166,\n",
              "    'precision': 0.9276315789473685,\n",
              "    'recall': 0.8493975903614458,\n",
              "    'spurious': 9},\n",
              "   'exact': {'actual': 152,\n",
              "    'correct': 140,\n",
              "    'incorrect': 3,\n",
              "    'missed': 23,\n",
              "    'partial': 0,\n",
              "    'possible': 166,\n",
              "    'precision': 0.9210526315789473,\n",
              "    'recall': 0.8433734939759037,\n",
              "    'spurious': 9},\n",
              "   'partial': {'actual': 149,\n",
              "    'correct': 140,\n",
              "    'incorrect': 0,\n",
              "    'missed': 23,\n",
              "    'partial': 3,\n",
              "    'possible': 166,\n",
              "    'precision': 0.9496644295302014,\n",
              "    'recall': 0.8524096385542169,\n",
              "    'spurious': 9},\n",
              "   'strict': {'actual': 152,\n",
              "    'correct': 138,\n",
              "    'incorrect': 5,\n",
              "    'missed': 23,\n",
              "    'partial': 0,\n",
              "    'possible': 166,\n",
              "    'precision': 0.9078947368421053,\n",
              "    'recall': 0.8313253012048193,\n",
              "    'spurious': 9}},\n",
              "  'POG': {'ent_type': {'actual': 12,\n",
              "    'correct': 8,\n",
              "    'incorrect': 4,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 16,\n",
              "    'precision': 0.6666666666666666,\n",
              "    'recall': 0.5,\n",
              "    'spurious': 0},\n",
              "   'exact': {'actual': 12,\n",
              "    'correct': 11,\n",
              "    'incorrect': 1,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 16,\n",
              "    'precision': 0.9166666666666666,\n",
              "    'recall': 0.6875,\n",
              "    'spurious': 0},\n",
              "   'partial': {'actual': 11,\n",
              "    'correct': 11,\n",
              "    'incorrect': 0,\n",
              "    'missed': 4,\n",
              "    'partial': 1,\n",
              "    'possible': 16,\n",
              "    'precision': 1.0454545454545454,\n",
              "    'recall': 0.71875,\n",
              "    'spurious': 0},\n",
              "   'strict': {'actual': 12,\n",
              "    'correct': 7,\n",
              "    'incorrect': 5,\n",
              "    'missed': 4,\n",
              "    'partial': 0,\n",
              "    'possible': 16,\n",
              "    'precision': 0.5833333333333334,\n",
              "    'recall': 0.4375,\n",
              "    'spurious': 0}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NuMMNCYh0bci"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0B4xeUHnNX53"
      },
      "source": [
        "### Coverage of tokens in test set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ek4aeAPLQO4e"
      },
      "source": [
        "#### Twiner"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYC0cBq9NbWc"
      },
      "source": [
        "testset_named_entities = []\n",
        "for tweet, tweet_tags in zip(texts, tags):\n",
        "  for word, tag in zip(tweet, tweet_tags):\n",
        "    if tag != 'O':\n",
        "      testset_named_entities.append(word)"
      ],
      "execution_count": 119,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pUJFCJfJOiBV"
      },
      "source": [
        "trainset_named_entities = []\n",
        "for tweet, tweet_tags in zip(texts_train, tags_train):\n",
        "  for word, tag in zip(tweet, tweet_tags):\n",
        "    if tag != 'O':\n",
        "      trainset_named_entities.append(word)"
      ],
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2L9bupQOjVa"
      },
      "source": [
        "diff_test_train = set(testset_named_entities) - set(trainset_named_entities)"
      ],
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g0Zqq6tYaMqc",
        "outputId": "9b2f77bb-4977-40d4-da3e-621c10a5d932"
      },
      "source": [
        "len(set(diff_test_train))"
      ],
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "147"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QCicFujda5TV",
        "outputId": "0bda101c-466c-456d-fb3e-c6abbd05279f"
      },
      "source": [
        "diff_test_train"
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'1987',\n",
              " 'Arthur',\n",
              " 'Childress',\n",
              " 'Demarest',\n",
              " 'afc',\n",
              " 'آشور',\n",
              " 'آکوامن',\n",
              " 'آیتی',\n",
              " 'ابا',\n",
              " 'ابوالفتوح',\n",
              " 'ابومرزوق',\n",
              " 'اردوگاه',\n",
              " 'ارومچیان',\n",
              " 'اسوشیتدپرس',\n",
              " 'اسپیس',\n",
              " 'اصلاحطلبها',\n",
              " 'افسانه',\n",
              " 'الوندی',\n",
              " 'الیگودرز',\n",
              " 'ال\\u200cسی',\n",
              " 'امداد',\n",
              " 'امریکائی\\u200cها',\n",
              " 'امپریالیستی',\n",
              " 'اهی',\n",
              " 'اوتادی',\n",
              " 'اینترسکشنالیتی',\n",
              " 'ایندیانا',\n",
              " 'ایکس',\n",
              " 'بخارا',\n",
              " 'بدوایم',\n",
              " 'برنا',\n",
              " 'بعث',\n",
              " 'بنان',\n",
              " 'بهارمست',\n",
              " 'بهرگان',\n",
              " 'بکا',\n",
              " 'بیرانشهر',\n",
              " 'بیروت',\n",
              " 'بیشاپور',\n",
              " 'بی\\u200cآی',\n",
              " 'بی\\u200c\\u200cبی\\u200cسی',\n",
              " 'تاثیرگذار\\u200c',\n",
              " 'تاپ',\n",
              " 'تریکو',\n",
              " 'تسوایک',\n",
              " 'تورکییسم',\n",
              " 'ج',\n",
              " 'جوانان',\n",
              " 'حکیمی',\n",
              " 'خاچ',\n",
              " 'خرمشاهی',\n",
              " 'دادستانی',\n",
              " 'دانش\\u200cپژوهان',\n",
              " 'دسته',\n",
              " 'دور',\n",
              " 'دیفکتو',\n",
              " 'دیویدسون',\n",
              " 'رافائل',\n",
              " 'راونسبروک',\n",
              " 'رختیانی',\n",
              " 'رضوان',\n",
              " 'رهبرو',\n",
              " 'روتسکوی',\n",
              " 'رومشکان',\n",
              " 'سالیانی',\n",
              " 'ساکاشویلی',\n",
              " 'سبز\\u200cوار',\n",
              " 'سلین',\n",
              " 'سندالی',\n",
              " 'سورآبادی',\n",
              " 'سورب',\n",
              " 'سویگرت',\n",
              " 'سپیدنام',\n",
              " 'سیدرضی',\n",
              " 'سیدعباسی',\n",
              " 'شاپ\\u200cکار',\n",
              " 'شفرو',\n",
              " 'شهدای',\n",
              " 'شیخی',\n",
              " 'شیرکوند',\n",
              " 'صابر',\n",
              " 'طبرسی',\n",
              " 'عثمان',\n",
              " 'عفوبین\\u200cالملل',\n",
              " 'فاطیما',\n",
              " 'فامیل\\u200cدشتی',\n",
              " 'فراز',\n",
              " 'فرانسیسکو',\n",
              " 'فرانس\\u200cپرنس',\n",
              " 'فردینان',\n",
              " 'لئونید',\n",
              " 'لهراسبی',\n",
              " 'لوترکینگوم',\n",
              " 'لونزدل',\n",
              " 'مارسیا',\n",
              " 'مانه\\u200cای',\n",
              " 'مانوئل',\n",
              " 'محمدآباد',\n",
              " 'مسکن',\n",
              " 'مسگرها',\n",
              " 'معادن',\n",
              " 'مقامی',\n",
              " 'ممنون',\n",
              " 'مور',\n",
              " 'موسسه\\u200cی',\n",
              " 'موموآ',\n",
              " 'مون\\u200cریکر',\n",
              " 'میلان\\u200cبودم',\n",
              " 'میلوشوویچ',\n",
              " 'میکونوس',\n",
              " 'نارنجی',\n",
              " 'نقش',\n",
              " 'ننه',\n",
              " 'نوتردام',\n",
              " 'نوجوانان',\n",
              " 'نیری',\n",
              " 'هارلی',\n",
              " 'هاشمی\\u200cزاده',\n",
              " 'هاوی',\n",
              " 'هداوند',\n",
              " 'همایی',\n",
              " 'هوستون',\n",
              " 'والاس',\n",
              " 'وسپا',\n",
              " 'وکیلی\\u200cراد',\n",
              " 'پایه',\n",
              " 'پرسا',\n",
              " 'پلدختر',\n",
              " 'پولادی',\n",
              " 'پولیتیکو',\n",
              " 'پوما',\n",
              " 'پونکو',\n",
              " 'چارلی',\n",
              " 'چنل',\n",
              " 'کاتالان',\n",
              " 'کرب\\u200cوبلا',\n",
              " 'کلاله',\n",
              " 'کلامبوس',\n",
              " 'کلاپر',\n",
              " 'کمپین',\n",
              " 'کومی',\n",
              " 'کوچما',\n",
              " 'گارین',\n",
              " 'گاگیک',\n",
              " 'گردآفرید',\n",
              " 'گرمارودی',\n",
              " 'گزاری'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 131
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KVABeNkHOzWE",
        "outputId": "6145a1e7-dafb-4fd6-8603-0aa19150d6a9"
      },
      "source": [
        "len(set(testset_named_entities))"
      ],
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "702"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzF5NXd5O8WE"
      },
      "source": [
        "diff_tweet = []\n",
        "diff_tags = []\n",
        "for tweet, tweet_tags in zip(texts, tags):\n",
        "  flag = False\n",
        "  for word, tag in zip(tweet, tweet_tags):\n",
        "    if tag != 'O':\n",
        "      if word not in diff_test_train:\n",
        "        flag = True\n",
        "  if flag == False:\n",
        "    diff_tweet.append(tweet)\n",
        "    diff_tags.append(tweet_tags)"
      ],
      "execution_count": 136,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qt7RRG8XPrpV",
        "outputId": "28189cfe-deba-408f-d50b-5eb2c777dd98"
      },
      "source": [
        "len(diff_tweet)"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "87"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZyRjssHDPttm",
        "outputId": "4833ce46-be33-46ac-f52c-bcebeae7ed93"
      },
      "source": [
        "len(texts)"
      ],
      "execution_count": 138,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "303"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_N6h_e93aV5l",
        "outputId": "71d73b88-3683-40e6-9fce-88e369a11817"
      },
      "source": [
        "diff_tweet"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['دلم',\n",
              "  'برای',\n",
              "  'شب\\u200cهای',\n",
              "  'میدان',\n",
              "  'تنگ',\n",
              "  'شده',\n",
              "  '.',\n",
              "  'خدایا',\n",
              "  'کرونا',\n",
              "  'نمیخواد',\n",
              "  'بره'],\n",
              " ['بله',\n",
              "  'و',\n",
              "  'بسیار',\n",
              "  'کار',\n",
              "  'خوبیه',\n",
              "  'اما',\n",
              "  'ادعای',\n",
              "  'پزشکان',\n",
              "  'و',\n",
              "  'دانشمندان',\n",
              "  'بسیار',\n",
              "  'بیشتر',\n",
              "  'از',\n",
              "  'این',\n",
              "  'حرفا',\n",
              "  'ست',\n",
              "  'بسیار',\n",
              "  'بسیار',\n",
              "  'کند',\n",
              "  'عمل',\n",
              "  'میکنن'],\n",
              " ['گفتم', 'با', 'خبرین', 'لابد'],\n",
              " ['منم',\n",
              "  'پرتغال',\n",
              "  'هرچی',\n",
              "  'میوه',\n",
              "  'باشه',\n",
              "  'پرتغال',\n",
              "  'نباشه',\n",
              "  'میگم',\n",
              "  'هیچی',\n",
              "  'میوه',\n",
              "  'نداریم',\n",
              "  'که'],\n",
              " ['ما',\n",
              "  'مرد',\n",
              "  'جنگیم',\n",
              "  'چه',\n",
              "  'جنگ',\n",
              "  'سخت',\n",
              "  'باشد',\n",
              "  'و',\n",
              "  'چه',\n",
              "  'جنگ',\n",
              "  'نرم',\n",
              "  'تصویر',\n",
              "  'مربوط',\n",
              "  'به',\n",
              "  'کانال',\n",
              "  'شهداست',\n",
              "  'که',\n",
              "  'به',\n",
              "  'مناسبت',\n",
              "  '40',\n",
              "  'سالگرد',\n",
              "  'تجاوز',\n",
              "  'دشمنان',\n",
              "  'به',\n",
              "  'میهن',\n",
              "  'اسلامی',\n",
              "  'و',\n",
              "  'دفاع',\n",
              "  'جانانه',\n",
              "  'رزمندگان',\n",
              "  'در',\n",
              "  'برابر',\n",
              "  'آنان',\n",
              "  'برای',\n",
              "  'اولین',\n",
              "  'بار',\n",
              "  'در',\n",
              "  'شهرستان',\n",
              "  'کلاله',\n",
              "  'ساخته',\n",
              "  'شده',\n",
              "  'است',\n",
              "  'شهدا',\n",
              "  'کانال',\n",
              "  'شهدا',\n",
              "  'بصیرت'],\n",
              " ['تو',\n",
              "  'را',\n",
              "  'بخدا',\n",
              "  'بگذارید',\n",
              "  'خود',\n",
              "  'مردم',\n",
              "  'تصمیم',\n",
              "  'بگیرند',\n",
              "  'مردم',\n",
              "  'صلاح',\n",
              "  'خود',\n",
              "  'را',\n",
              "  'بهتر',\n",
              "  'از',\n",
              "  'هر',\n",
              "  'کسی',\n",
              "  'می\\u200cدانند',\n",
              "  'شماها',\n",
              "  'به',\n",
              "  'وظائف',\n",
              "  'نمایندگی',\n",
              "  'خود',\n",
              "  'بپردازید',\n",
              "  'اکثریت',\n",
              "  'هر',\n",
              "  'چه',\n",
              "  'گفت',\n",
              "  'بزارید',\n",
              "  'همان',\n",
              "  'شود',\n",
              "  'حتی',\n",
              "  'اگر',\n",
              "  'بر',\n",
              "  'ضرر',\n",
              "  'خودشان',\n",
              "  'باشد',\n",
              "  'شما',\n",
              "  'وکیل',\n",
              "  'آن\\u200cها',\n",
              "  'هستید',\n",
              "  'ولی',\n",
              "  'آن\\u200cها',\n",
              "  'نیستید'],\n",
              " ['گاهی', 'اگر', 'آهی', 'کشم', 'دلها', 'بسوزد'],\n",
              " ['آره',\n",
              "  'حق\\u200cداری',\n",
              "  'مسخره',\n",
              "  'مون',\n",
              "  'کن',\n",
              "  'ما',\n",
              "  'حق',\n",
              "  'نداریم',\n",
              "  'همونجور',\n",
              "  'که',\n",
              "  'نماینده\\u200cهامون',\n",
              "  'به',\n",
              "  'خودرو',\n",
              "  'میرسن',\n",
              "  '؛',\n",
              "  'برسیم',\n",
              "  'آخه',\n",
              "  'اونا',\n",
              "  'خونشون',\n",
              "  'رنگینه',\n",
              "  'خدا',\n",
              "  'نجاتت',\n",
              "  'بده'],\n",
              " ['برای',\n",
              "  'جنازه\\u200cی',\n",
              "  'از',\n",
              "  'کشیک',\n",
              "  'برگشته\\u200cای',\n",
              "  'که',\n",
              "  'مستقیم',\n",
              "  'از',\n",
              "  'بیمارستان',\n",
              "  'داره',\n",
              "  'میره',\n",
              "  'تو',\n",
              "  'جاده',\n",
              "  'به',\n",
              "  'سمت',\n",
              "  'شمال',\n",
              "  'اهنگ',\n",
              "  'شاد',\n",
              "  'اوپوس',\n",
              "  'اوپوس',\n",
              "  'بفرستید',\n",
              "  'لطفا',\n",
              "  'اگر',\n",
              "  'مقدوره',\n",
              "  'سپاسمندم'],\n",
              " ['اهنگ',\n",
              "  'امشب',\n",
              "  'دایه',\n",
              "  'دایه',\n",
              "  'وقت',\n",
              "  'جنگه',\n",
              "  'از',\n",
              "  'گروه',\n",
              "  'سرود',\n",
              "  'کوهستان',\n",
              "  'قبل',\n",
              "  'این',\n",
              "  'که',\n",
              "  'بیاید',\n",
              "  'بگید',\n",
              "  'سقایی',\n",
              "  'باید',\n",
              "  'بگم',\n",
              "  'خودم',\n",
              "  'بلدم',\n",
              "  'ولی',\n",
              "  'این',\n",
              "  'باحالتره'],\n",
              " ['تصور',\n",
              "  'کنید',\n",
              "  'اندامی',\n",
              "  'از',\n",
              "  '«',\n",
              "  'یک',\n",
              "  'پیکر',\n",
              "  '»',\n",
              "  'بزرگ\\u200cتر',\n",
              "  'هستید',\n",
              "  'و',\n",
              "  'فیلمی',\n",
              "  'مثلا',\n",
              "  'مستند',\n",
              "  'می\\u200cبینید',\n",
              "  'درباره\\u200cی',\n",
              "  'عفونتی',\n",
              "  'که',\n",
              "  'این',\n",
              "  '«',\n",
              "  'پیکر',\n",
              "  '»',\n",
              "  'را',\n",
              "  'مبتلا',\n",
              "  'کرده',\n",
              "  'و',\n",
              "  'هیچ',\n",
              "  'اندامی',\n",
              "  'از',\n",
              "  'جمله',\n",
              "  'شما',\n",
              "  'از',\n",
              "  'مرگباری',\n",
              "  'آن',\n",
              "  'در',\n",
              "  'امان',\n",
              "  'نیستید',\n",
              "  'فیلمساز',\n",
              "  'یک',\n",
              "  'صبح',\n",
              "  'تا',\n",
              "  'بعدازظهر',\n",
              "  'دنبال',\n",
              "  'یکی',\n",
              "  'از',\n",
              "  'این',\n",
              "  'اندام\\u200cهای',\n",
              "  'عفونی',\n",
              "  'راه',\n",
              "  'افتاده',\n",
              "  'و',\n",
              "  'زندگی\\u200cاش',\n",
              "  'را',\n",
              "  'نشان',\n",
              "  'می\\u200cدهد'],\n",
              " ['ممنونم', 'از', 'آموزش', 'خوبتون', 'کاملا', 'منطقیه'],\n",
              " ['اصلا',\n",
              "  'بفهمند',\n",
              "  'کارگر',\n",
              "  'هستی',\n",
              "  'برخوردا',\n",
              "  'زننده',\n",
              "  'تر',\n",
              "  'میشه',\n",
              "  'بفهمند',\n",
              "  'شهرستانی',\n",
              "  'هستی',\n",
              "  'بدتر',\n",
              "  'بفهمند',\n",
              "  'از',\n",
              "  'خانواده',\n",
              "  'ضعیف',\n",
              "  'بدتر',\n",
              "  'و',\n",
              "  'بدتر'],\n",
              " ['علت',\n",
              "  'این',\n",
              "  'اشتیاق',\n",
              "  'مشخص',\n",
              "  'است',\n",
              "  'مردم',\n",
              "  'به',\n",
              "  'فروپاشی',\n",
              "  'نظام\\u200cهای',\n",
              "  'توتالیتر',\n",
              "  'علاقمندند',\n",
              "  'بویژه',\n",
              "  'آن\\u200cها',\n",
              "  'که',\n",
              "  'داعیه',\n",
              "  'خدمت',\n",
              "  'به',\n",
              "  'آرمانی',\n",
              "  '(',\n",
              "  'محرومان',\n",
              "  'اسلام',\n",
              "  ')',\n",
              "  'را',\n",
              "  'داشته\\u200cاند',\n",
              "  'و',\n",
              "  'شناخت',\n",
              "  'این',\n",
              "  'نظامها',\n",
              "  'و',\n",
              "  'ویژگی',\n",
              "  'آن\\u200cها',\n",
              "  'که',\n",
              "  'به',\n",
              "  'قیام',\n",
              "  'سرکوب',\n",
              "  'شدگان',\n",
              "  'انجامیده',\n",
              "  'همیشه',\n",
              "  'راهگشاست'],\n",
              " ['کره',\n",
              "  'مضر\\u200cبود',\n",
              "  'گرانش',\n",
              "  'کردیم',\n",
              "  'پفک',\n",
              "  'که',\n",
              "  'جای',\n",
              "  'خودش',\n",
              "  'را',\n",
              "  'دارد',\n",
              "  'پفک',\n",
              "  'هزاری',\n",
              "  'را',\n",
              "  'پنج',\n",
              "  'هزار',\n",
              "  'می\\u200cکنیم',\n",
              "  'و',\n",
              "  'چقدر',\n",
              "  'سلامت',\n",
              "  'مردممان',\n",
              "  'همچون',\n",
              "  'عملکردمان',\n",
              "  'در',\n",
              "  'کرونا',\n",
              "  'برایمان',\n",
              "  'مهم',\n",
              "  'است',\n",
              "  'تحریم',\n",
              "  'یا',\n",
              "  'مدیریت',\n",
              "  'ناکارآمد'],\n",
              " ['یه', 'استقلالی', 'برنده', 'با', 'معرفت', 'بهتون', 'تبریک', 'میگه'],\n",
              " ['وقتی',\n",
              "  'به',\n",
              "  'کسی',\n",
              "  'کمک',\n",
              "  'می\\u200cکنیم',\n",
              "  'ودلی',\n",
              "  'راشاد',\n",
              "  'می\\u200cکنیم',\n",
              "  'لبخندرضایت',\n",
              "  'خدانوری',\n",
              "  'رادر',\n",
              "  'دل',\n",
              "  'ماایجاد',\n",
              "  'میکندکه',\n",
              "  'شادی',\n",
              "  'ونشاط',\n",
              "  'عمیقی',\n",
              "  'درخودمون',\n",
              "  'احساس',\n",
              "  'می\\u200cکنیم',\n",
              "  'اگر',\n",
              "  'خودمونو',\n",
              "  'دوست',\n",
              "  'داریم',\n",
              "  'ومیخایم',\n",
              "  'همیشه',\n",
              "  'شاد',\n",
              "  'باشیم',\n",
              "  'درشادکردن',\n",
              "  'دیگران',\n",
              "  'بکوشیم'],\n",
              " ['مهمترین',\n",
              "  'و',\n",
              "  'اولین',\n",
              "  'انتظار',\n",
              "  'تصویب',\n",
              "  'طرح',\n",
              "  'شفافیت',\n",
              "  'ارا',\n",
              "  'نمایندگان',\n",
              "  'تا',\n",
              "  'کارشناسان',\n",
              "  'دلسوز',\n",
              "  'کشور',\n",
              "  'دقیقتر',\n",
              "  'مطالبه',\n",
              "  'گری',\n",
              "  'کنند',\n",
              "  'و',\n",
              "  'در',\n",
              "  'نتیجه',\n",
              "  'سواد',\n",
              "  'سیاسی',\n",
              "  'و',\n",
              "  'اقتصادی',\n",
              "  'مردم',\n",
              "  'بالاتر',\n",
              "  'رود',\n",
              "  'و',\n",
              "  'قانون\\u200cگذاری',\n",
              "  'برای',\n",
              "  'نابودی',\n",
              "  'فساد',\n",
              "  'و',\n",
              "  'حل',\n",
              "  'مشکلات',\n",
              "  'به',\n",
              "  'صورت',\n",
              "  'ریشه\\u200cای',\n",
              "  'اغاز',\n",
              "  'شود'],\n",
              " ['تو',\n",
              "  'یه',\n",
              "  'دنیای',\n",
              "  'موازی',\n",
              "  'به',\n",
              "  'جای',\n",
              "  'جنگ',\n",
              "  'برای',\n",
              "  'نمره',\n",
              "  'یه',\n",
              "  'دانشجوی',\n",
              "  'هنر',\n",
              "  'تو',\n",
              "  'میلان\\u200cبودم',\n",
              "  'و',\n",
              "  'موزه\\u200cهای',\n",
              "  'مختلف',\n",
              "  'رو',\n",
              "  'می\\u200cگشتم',\n",
              "  'که',\n",
              "  'الهام',\n",
              "  'بگیرم',\n",
              "  'برا',\n",
              "  'نقاشیام',\n",
              "  '،',\n",
              "  'تنها',\n",
              "  'دغدغه\\u200cم',\n",
              "  'هم',\n",
              "  'این\\u200cبود',\n",
              "  'که',\n",
              "  'این',\n",
              "  'قلمروی',\n",
              "  'موی',\n",
              "  'سمور',\n",
              "  'که',\n",
              "  'دارم',\n",
              "  'حقوق',\n",
              "  'حیوانات',\n",
              "  'رو',\n",
              "  'نقض',\n",
              "  'می\\u200cکنه',\n",
              "  '.'],\n",
              " ['الکی',\n",
              "  'بولد',\n",
              "  '\\u200cنشده',\n",
              "  '\\u200cروش',\n",
              "  'بازاریابی',\n",
              "  'هست',\n",
              "  'چیزی',\n",
              "  'به',\n",
              "  'مخاطب',\n",
              "  'بده',\n",
              "  'که',\n",
              "  'متعجبش',\n",
              "  'کنه',\n",
              "  '!'],\n",
              " ['وسپا',\n",
              "  'خیلی',\n",
              "  'دخترونس',\n",
              "  ':)',\n",
              "  'پسر',\n",
              "  'باید',\n",
              "  'هارلی',\n",
              "  'دیویدسون',\n",
              "  'سوار',\n",
              "  'باشه'],\n",
              " ['نقش', 'حال\\u200cحاضر', 'من', 'در', 'خانواده', 'مترسک', 'کرونا'],\n",
              " ['وارونه\\u200cگویی',\n",
              "  'و',\n",
              "  'وارونه\\u200cنمایی',\n",
              "  'حتی',\n",
              "  'برای',\n",
              "  'زندانهای',\n",
              "  'قرون',\n",
              "  'وسطایی',\n",
              "  '!'],\n",
              " ['شما',\n",
              "  'سعی',\n",
              "  'کن',\n",
              "  'کالسکه',\n",
              "  'رو',\n",
              "  'بذاری',\n",
              "  'صندوق',\n",
              "  'سدان',\n",
              "  'بعد',\n",
              "  'متوجه',\n",
              "  'دلیل',\n",
              "  'محبوبیتش',\n",
              "  'میشی',\n",
              "  '.'],\n",
              " ['اگر',\n",
              "  'گفتمان',\n",
              "  'مقابل',\n",
              "  'این',\n",
              "  'اندیشه',\n",
              "  'هم',\n",
              "  'بر',\n",
              "  'سر',\n",
              "  'کار\\u200cبود',\n",
              "  'تضمینی',\n",
              "  'نبود',\n",
              "  'که',\n",
              "  'بتونه',\n",
              "  'در',\n",
              "  'گذر',\n",
              "  'از',\n",
              "  'شرایط',\n",
              "  'و',\n",
              "  'موقعیت\\u200cهای',\n",
              "  'این',\n",
              "  'هفت',\n",
              "  'سال',\n",
              "  'مفتضح',\n",
              "  'نشه',\n",
              "  'و',\n",
              "  'رضایت',\n",
              "  'مردم',\n",
              "  'رو',\n",
              "  'بدست',\n",
              "  'بیاره',\n",
              "  '،',\n",
              "  '(',\n",
              "  'که',\n",
              "  'در',\n",
              "  'هشت',\n",
              "  'سال',\n",
              "  'قبلی',\n",
              "  'دست',\n",
              "  'کم',\n",
              "  'موقعیت',\n",
              "  'مشابهی',\n",
              "  'یافت',\n",
              "  '!',\n",
              "  ')',\n",
              "  'به',\n",
              "  'هر',\n",
              "  'روی',\n",
              "  'کار',\n",
              "  'مملکت',\n",
              "  'با',\n",
              "  'ریشه\\u200cیابی',\n",
              "  'مشکلات',\n",
              "  'و',\n",
              "  'اصلاح',\n",
              "  'به',\n",
              "  'نتیجه',\n",
              "  'میرسه',\n",
              "  'نه',\n",
              "  'این',\n",
              "  'قبیل',\n",
              "  'لعن\\u200cها',\n",
              "  '!'],\n",
              " ['دلایل',\n",
              "  'متعددی',\n",
              "  'ممکنه',\n",
              "  'داشته',\n",
              "  'باشد',\n",
              "  '.',\n",
              "  'مهمترینهاش',\n",
              "  'اینها',\n",
              "  'هستند',\n",
              "  '1کسی',\n",
              "  'که',\n",
              "  'در',\n",
              "  'یک',\n",
              "  'مجله',\n",
              "  'هیات',\n",
              "  'تحریریه',\n",
              "  'است',\n",
              "  'معمولا',\n",
              "  'در',\n",
              "  'مجله',\n",
              "  'دیگر',\n",
              "  'قدرت',\n",
              "  'بده',\n",
              "  'بستان',\n",
              "  'دارد',\n",
              "  '.',\n",
              "  '2',\n",
              "  'وجود',\n",
              "  'اسم',\n",
              "  'یک',\n",
              "  'ادم',\n",
              "  'معتبر',\n",
              "  'احتمال',\n",
              "  'پدیرش',\n",
              "  'را',\n",
              "  'بالا',\n",
              "  'میبرد',\n",
              "  '3سرعت',\n",
              "  'پدیرش',\n",
              "  'را',\n",
              "  'بالا',\n",
              "  'میبرد',\n",
              "  '.',\n",
              "  'در',\n",
              "  'هر',\n",
              "  'حال',\n",
              "  'کلا',\n",
              "  'کار',\n",
              "  'فساد',\n",
              "  'الودی',\n",
              "  'است',\n",
              "  '.'],\n",
              " ['من',\n",
              "  'یه',\n",
              "  'کتاب',\n",
              "  'دارم',\n",
              "  'تمرینای',\n",
              "  'ساده',\n",
              "  'ش',\n",
              "  'خوراک',\n",
              "  'گرم',\n",
              "  'و',\n",
              "  'نرم',\n",
              "  'کردن',\n",
              "  'بدنه',\n",
              "  '،',\n",
              "  'تمرینای',\n",
              "  'اونو',\n",
              "  'با',\n",
              "  'ژیمناستیک',\n",
              "  'قاطی',\n",
              "  'کنی',\n",
              "  'بدنت',\n",
              "  'معرکه',\n",
              "  'میشه'],\n",
              " ['امسال',\n",
              "  'برا',\n",
              "  'تولدم',\n",
              "  'یه',\n",
              "  'دور',\n",
              "  'آبجیم',\n",
              "  'اینا',\n",
              "  'سورپرایز',\n",
              "  'کردن',\n",
              "  'یبار',\n",
              "  'دوستام',\n",
              "  '،',\n",
              "  'هردوش',\n",
              "  'رو',\n",
              "  'از',\n",
              "  'قبل',\n",
              "  'فهمیدم',\n",
              "  'و',\n",
              "  'به',\n",
              "  'روشون',\n",
              "  'نیوردم',\n",
              "  '.',\n",
              "  'الان',\n",
              "  'فیلمم',\n",
              "  'رو',\n",
              "  'ببینید',\n",
              "  'میگید',\n",
              "  'دیگه',\n",
              "  'محاله',\n",
              "  'بشر',\n",
              "  'ذوق',\n",
              "  'زده\\u200cتر',\n",
              "  'از',\n",
              "  'این',\n",
              "  'بشه',\n",
              "  'خیلی',\n",
              "  'خوشحال',\n",
              "  'شدم',\n",
              "  'ولی',\n",
              "  'خب',\n",
              "  'اون',\n",
              "  'چیزی',\n",
              "  'که',\n",
              "  'دیدن',\n",
              "  'واقعا',\n",
              "  'نمایشی',\n",
              "  'بود',\n",
              "  '.'],\n",
              " ['تبدیل', 'مایع', 'به', 'گاز'],\n",
              " ['تا',\n",
              "  'پایمان',\n",
              "  'بیشتر',\n",
              "  'نسوزد',\n",
              "  '.',\n",
              "  'شب',\n",
              "  'که',\n",
              "  'میشد',\n",
              "  'پاها',\n",
              "  'تاول',\n",
              "  'زده',\n",
              "  'و',\n",
              "  'گزگز',\n",
              "  'میکرد',\n",
              "  '.',\n",
              "  'مادر',\n",
              "  'پارچه',\n",
              "  'میسوزاند',\n",
              "  'و',\n",
              "  'سوخته\\u200c',\n",
              "  'ان',\n",
              "  'را',\n",
              "  'روی',\n",
              "  'زخم',\n",
              "  'میگذاشت',\n",
              "  'تا',\n",
              "  'صبح',\n",
              "  'و',\n",
              "  'میگفت',\n",
              "  'که',\n",
              "  'مرهم',\n",
              "  'است',\n",
              "  '.',\n",
              "  'اما',\n",
              "  'مادر',\n",
              "  'خودش',\n",
              "  'مرهم',\n",
              "  'زندگی',\n",
              "  'پر',\n",
              "  'مشّقت',\n",
              "  'ما\\u200c',\n",
              "  'بود',\n",
              "  '.',\n",
              "  'زندگی',\n",
              "  'در',\n",
              "  'فقر',\n",
              "  'و',\n",
              "  'نداری',\n",
              "  'او',\n",
              "  'را',\n",
              "  'مثل',\n",
              "  'همان',\n",
              "  'پارچه\\u200cها',\n",
              "  'سوزانده',\n",
              "  'بود',\n",
              "  '.'],\n",
              " ['آره',\n",
              "  'واقعا',\n",
              "  'سخته',\n",
              "  '،',\n",
              "  'ولی',\n",
              "  'بعضی\\u200cها',\n",
              "  'گویا',\n",
              "  'موفق',\n",
              "  'میشن',\n",
              "  'اگه',\n",
              "  'وانمود',\n",
              "  'نکنند',\n",
              "  'رابطه\\u200cی',\n",
              "  'نزدیکی',\n",
              "  'داشته',\n",
              "  'باشن',\n",
              "  '.',\n",
              "  'من',\n",
              "  'فکر',\n",
              "  'می\\u200cکنم',\n",
              "  'وقتی',\n",
              "  'از',\n",
              "  'نزدیک',\n",
              "  'تو',\n",
              "  'زندگی',\n",
              "  'هم',\n",
              "  'درگیر',\n",
              "  'نباشن',\n",
              "  'آدم\\u200cها',\n",
              "  'حرف',\n",
              "  'زیادی',\n",
              "  'برای',\n",
              "  'زدن',\n",
              "  'نمی\\u200cمونه',\n",
              "  '.'],\n",
              " ['این',\n",
              "  'گراف',\n",
              "  'رو',\n",
              "  'به',\n",
              "  'یک',\n",
              "  'متخصص',\n",
              "  'رادیولوژی',\n",
              "  'داممزشکی',\n",
              "  'یا',\n",
              "  'داخلی',\n",
              "  'دام',\n",
              "  'کوچک',\n",
              "  'نشان',\n",
              "  'دهید',\n",
              "  '.',\n",
              "  'به',\n",
              "  'نظر',\n",
              "  'مشکل',\n",
              "  'فقط',\n",
              "  'شکستگی',\n",
              "  'نیست'],\n",
              " ['الان',\n",
              "  'یهو',\n",
              "  'یادم',\n",
              "  'افتاد',\n",
              "  'وقتی',\n",
              "  'بچه',\n",
              "  'بودم',\n",
              "  'رو',\n",
              "  'فاطیما',\n",
              "  'بهارمست',\n",
              "  'کراش',\n",
              "  'شدید',\n",
              "  'داشتم',\n",
              "  ':))))))))'],\n",
              " ['این',\n",
              "  'که',\n",
              "  'جوونه',\n",
              "  'هنوز',\n",
              "  ':-))))',\n",
              "  'حالا',\n",
              "  'حالاها',\n",
              "  'آب',\n",
              "  'حیات',\n",
              "  'نمی\\u200cخواد',\n",
              "  '.'],\n",
              " ['سیاره',\n",
              "  '\"',\n",
              "  'عدد',\n",
              "  'پی',\n",
              "  '\"',\n",
              "  'کشف',\n",
              "  'شد',\n",
              "  '.',\n",
              "  'اخترشناسان',\n",
              "  'یک',\n",
              "  'سیاره',\n",
              "  'فراخورشیدی',\n",
              "  'جدید',\n",
              "  'کشف',\n",
              "  'کرده\\u200cاند',\n",
              "  'که',\n",
              "  'به',\n",
              "  'دلیل',\n",
              "  'اینکه',\n",
              "  'با',\n",
              "  'سرعت',\n",
              "  '3.14',\n",
              "  'برابر',\n",
              "  'زمین',\n",
              "  'به',\n",
              "  'دور',\n",
              "  'خورشید',\n",
              "  'خود',\n",
              "  'می\\u200cچرخد',\n",
              "  '،',\n",
              "  'آن',\n",
              "  'را',\n",
              "  'سیاره',\n",
              "  'عدد',\n",
              "  'پی',\n",
              "  'لقب',\n",
              "  'داده\\u200cاند',\n",
              "  '.'],\n",
              " ['نمیگم',\n",
              "  'ادمهای',\n",
              "  'بدی\\u200cان',\n",
              "  'ولی',\n",
              "  'خیلی',\n",
              "  'سخته',\n",
              "  'اعتماد',\n",
              "  'کردن',\n",
              "  'وگرنه',\n",
              "  'از',\n",
              "  'نزدیکانم',\n",
              "  'رفتن',\n",
              "  'و',\n",
              "  'خیلی',\n",
              "  'از',\n",
              "  'مهمون\\u200cنوازی\\u200cهاشون',\n",
              "  'تعریف',\n",
              "  'میکنن',\n",
              "  'انشاالله',\n",
              "  'سال',\n",
              "  'دیگه',\n",
              "  'راه',\n",
              "  'باز',\n",
              "  'شه',\n",
              "  'منم',\n",
              "  'باهاتون',\n",
              "  'بیام',\n",
              "  '.',\n",
              "  'هرچی',\n",
              "  'باشه',\n",
              "  'یه',\n",
              "  'تجربه',\n",
              "  'ی',\n",
              "  'جدیده',\n",
              "  'هیجان',\n",
              "  'داره'],\n",
              " ['ما',\n",
              "  'پدران',\n",
              "  'غیورمان',\n",
              "  'را',\n",
              "  'از',\n",
              "  'یاد',\n",
              "  'بردیم',\n",
              "  '.',\n",
              "  'پدرانی',\n",
              "  'که',\n",
              "  'بر',\n",
              "  'شرق',\n",
              "  'و',\n",
              "  'غرب',\n",
              "  'عالم',\n",
              "  'حکومت',\n",
              "  'میکردند',\n",
              "  '.',\n",
              "  'ولی',\n",
              "  'انها',\n",
              "  'هرگز',\n",
              "  'فراموش',\n",
              "  'نکرده\\u200cاند',\n",
              "  'که',\n",
              "  'ما',\n",
              "  'گرگزاده',\n",
              "  'ایم',\n",
              "  '.',\n",
              "  'فرصت',\n",
              "  'پیدا',\n",
              "  'کنیم',\n",
              "  'تمام',\n",
              "  'دنیا',\n",
              "  'جلودار',\n",
              "  'ما',\n",
              "  'نخواهد\\u200cبود',\n",
              "  '.',\n",
              "  'لذا',\n",
              "  'هر',\n",
              "  'چه',\n",
              "  'در',\n",
              "  'توان',\n",
              "  'دارند',\n",
              "  'ما',\n",
              "  'را',\n",
              "  'میکوبند',\n",
              "  '.',\n",
              "  'اینده',\n",
              "  'مال',\n",
              "  'ترکان',\n",
              "  'است',\n",
              "  '.'],\n",
              " ['سخنان',\n",
              "  'نگران\\u200cکننده',\n",
              "  'دکتر',\n",
              "  'طبرسی',\n",
              "  'در',\n",
              "  'باره',\n",
              "  'مدیریت',\n",
              "  'بیماری',\n",
              "  'کرونا',\n",
              "  '.'],\n",
              " ['وقتی',\n",
              "  'به',\n",
              "  'این',\n",
              "  'فکر',\n",
              "  'میکنم',\n",
              "  'که',\n",
              "  'کلی',\n",
              "  'برنامه\\u200cریزی',\n",
              "  'کرده',\n",
              "  'بودیم',\n",
              "  'با',\n",
              "  'داداشم',\n",
              "  'عید',\n",
              "  'بریم',\n",
              "  'پیش',\n",
              "  'دایی',\n",
              "  'امید',\n",
              "  '،',\n",
              "  'واسمون',\n",
              "  'ازقصه\\u200cهای',\n",
              "  'بچگیمون',\n",
              "  'بگه',\n",
              "  'تا',\n",
              "  'داستان',\n",
              "  'زندگی',\n",
              "  'بابابزرگ',\n",
              "  'و',\n",
              "  'همه',\n",
              "  'رو',\n",
              "  'ضبط',\n",
              "  'کنیم',\n",
              "  '.',\n",
              "  'غصه',\n",
              "  'م',\n",
              "  'میگیره',\n",
              "  '.',\n",
              "  'از',\n",
              "  'نبودنش',\n",
              "  'غصه',\n",
              "  'م',\n",
              "  'میگیره',\n",
              "  '.',\n",
              "  'کاش',\n",
              "  'اینقد',\n",
              "  'راه',\n",
              "  'و',\n",
              "  'جاده',\n",
              "  'بینمون',\n",
              "  'نبود',\n",
              "  'دایی',\n",
              "  'جون',\n",
              "  '.',\n",
              "  'چقد',\n",
              "  'شنیدنیای',\n",
              "  'شنیده',\n",
              "  'نشده',\n",
              "  'با',\n",
              "  'دایی',\n",
              "  'پرواز',\n",
              "  'کرد',\n",
              "  '.'],\n",
              " ['صدها',\n",
              "  'سال',\n",
              "  'پیش',\n",
              "  'دانشکده',\n",
              "  'هنرهای',\n",
              "  'زیبا',\n",
              "  '.',\n",
              "  'یکی',\n",
              "  'از',\n",
              "  'کارهای',\n",
              "  'پروژه',\n",
              "  'پایانی',\n",
              "  'خودم'],\n",
              " ['این',\n",
              "  'چ',\n",
              "  'منطق',\n",
              "  'چرندی',\n",
              "  'بود',\n",
              "  '؟',\n",
              "  'لعنت',\n",
              "  'توهین',\n",
              "  'نیست',\n",
              "  'و',\n",
              "  'گفتن',\n",
              "  'تو',\n",
              "  'ملا',\n",
              "  'عام',\n",
              "  'انجام',\n",
              "  'نشه',\n",
              "  '؟',\n",
              "  'اصن',\n",
              "  'اخوند',\n",
              "  'مسجد',\n",
              "  'محلتون',\n",
              "  'همینو',\n",
              "  'گفت',\n",
              "  'پشتشم',\n",
              "  'گفت',\n",
              "  'امام',\n",
              "  'فلان',\n",
              "  'گفته',\n",
              "  '.',\n",
              "  'خود',\n",
              "  'تو',\n",
              "  'نباید',\n",
              "  'ی',\n",
              "  'ذره',\n",
              "  'فکر',\n",
              "  'کنی',\n",
              "  'با',\n",
              "  'خودت',\n",
              "  '؟',\n",
              "  'اگ',\n",
              "  'بده',\n",
              "  'دیگ',\n",
              "  'اشکار',\n",
              "  'و',\n",
              "  'پنهان',\n",
              "  'نداره',\n",
              "  'ک',\n",
              "  '.'],\n",
              " ['فاکتورهای',\n",
              "  'بیشماری',\n",
              "  'درش',\n",
              "  'نقش',\n",
              "  'دارن',\n",
              "  'که',\n",
              "  'شما',\n",
              "  'بهتر',\n",
              "  'از',\n",
              "  'من',\n",
              "  'میدونی',\n",
              "  'گرامی',\n",
              "  '،',\n",
              "  'سوسیالیسم',\n",
              "  'در',\n",
              "  'تقابل',\n",
              "  'با',\n",
              "  'نظام',\n",
              "  'سرمایه\\u200cداری',\n",
              "  '.',\n",
              "  'نظام',\n",
              "  'سرمایه\\u200cداری',\n",
              "  'با',\n",
              "  'تمام',\n",
              "  'ضعف',\n",
              "  'و',\n",
              "  'کمبودهاش',\n",
              "  'ثابت',\n",
              "  'کرده',\n",
              "  'در',\n",
              "  'حال',\n",
              "  'حاضر',\n",
              "  'بهترین',\n",
              "  'سیستمه',\n",
              "  '.',\n",
              "  'باشد',\n",
              "  'که',\n",
              "  'روزی',\n",
              "  'نظامی',\n",
              "  'بهتر',\n",
              "  'ارائه',\n",
              "  'بشه',\n",
              "  'ولی',\n",
              "  'با',\n",
              "  'هم',\n",
              "  'هم',\n",
              "  'عقیده\\u200cایم',\n",
              "  'که',\n",
              "  'سوسیالیزم',\n",
              "  'در',\n",
              "  'تقابل',\n",
              "  'سرمایه\\u200cداری',\n",
              "  'محکوم',\n",
              "  'بفنا',\n",
              "  'ست'],\n",
              " ['من', 'گوژ', 'پشت', 'نوتردام', 'میشم', 'ولی'],\n",
              " ['مگه', 'من', 'خاله', 'تو', 'نیستم'],\n",
              " ['میبینی',\n",
              "  'خانم',\n",
              "  'وکیل',\n",
              "  '،',\n",
              "  'آدم',\n",
              "  'با',\n",
              "  'گوشیشم',\n",
              "  'نمیتونه',\n",
              "  'خلوت',\n",
              "  'کنه',\n",
              "  'تو',\n",
              "  'دهکده',\n",
              "  'جهانی'],\n",
              " ['بابا',\n",
              "  'اینا',\n",
              "  'سواد',\n",
              "  'ندارن',\n",
              "  '.',\n",
              "  'براشون',\n",
              "  'ی',\n",
              "  'نهضت',\n",
              "  'سوادآموزی',\n",
              "  'بزاریم'],\n",
              " ['این',\n",
              "  'ینی',\n",
              "  'داره',\n",
              "  'در',\n",
              "  'کل',\n",
              "  'میگه',\n",
              "  'نباید',\n",
              "  'بیشتر',\n",
              "  'از',\n",
              "  'این',\n",
              "  '.',\n",
              "  'بخوامت',\n",
              "  'چون',\n",
              "  'ریسکه',\n",
              "  '.',\n",
              "  'یا',\n",
              "  'اصلا',\n",
              "  'همون',\n",
              "  'اول',\n",
              "  'آهنگ',\n",
              "  'اون',\n",
              "  'جا',\n",
              "  'که',\n",
              "  'میگه',\n",
              "  '\"',\n",
              "  'خیلی',\n",
              "  'درد',\n",
              "  'داره',\n",
              "  'که',\n",
              "  'تو',\n",
              "  'بیشتر',\n",
              "  'میخواستی',\n",
              "  'و',\n",
              "  'من',\n",
              "  'انجام',\n",
              "  'ندادم',\n",
              "  '\"',\n",
              "  'بازم',\n",
              "  'داره',\n",
              "  'به',\n",
              "  'این',\n",
              "  'اشاره',\n",
              "  'میکنه',\n",
              "  'که',\n",
              "  'داره',\n",
              "  'یه',\n",
              "  'جورایی',\n",
              "  'سعی',\n",
              "  'میکنه',\n",
              "  'ازش',\n",
              "  'دوری',\n",
              "  'کنه',\n",
              "  'تا',\n",
              "  'اون',\n",
              "  'تو',\n",
              "  'خطر',\n",
              "  'نیوفته',\n",
              "  '!',\n",
              "  '!',\n",
              "  'اصن',\n",
              "  'پشمام',\n",
              "  'با',\n",
              "  'این',\n",
              "  'موزیک',\n",
              "  'ریخت'],\n",
              " ['به',\n",
              "  'به',\n",
              "  'از',\n",
              "  'این',\n",
              "  'گل',\n",
              "  'شاه\\u200cپسندی',\n",
              "  '.',\n",
              "  '*',\n",
              "  'مونولوگ',\n",
              "  'پرویزخان',\n",
              "  'همراه',\n",
              "  'با',\n",
              "  'لذت',\n",
              "  'فراوان',\n",
              "  '.',\n",
              "  'در',\n",
              "  'مواجهه',\n",
              "  'با',\n",
              "  'گل',\n",
              "  'محبوبش',\n",
              "  'تو',\n",
              "  'حیاط',\n",
              "  '،',\n",
              "  'همین',\n",
              "  'الان',\n",
              "  '.',\n",
              "  'منم',\n",
              "  'کلی',\n",
              "  'خندیدم',\n",
              "  'و',\n",
              "  'ذوق',\n",
              "  'کردم',\n",
              "  'از',\n",
              "  'ذوقش',\n",
              "  '.',\n",
              "  'هنوزم',\n",
              "  'داره',\n",
              "  'کیف',\n",
              "  'می\\u200cکنه\\u200cها',\n",
              "  '.',\n",
              "  'روزمو',\n",
              "  'ساخت',\n",
              "  '.',\n",
              "  'به',\n",
              "  'به',\n",
              "  'از',\n",
              "  'این',\n",
              "  'گل',\n",
              "  'شاه\\u200cپسندی',\n",
              "  'رفقا'],\n",
              " ['«',\n",
              "  'سیاست',\n",
              "  'امپریالیستی',\n",
              "  'اینترسکشنالیتی',\n",
              "  '»',\n",
              "  'را',\n",
              "  'خواندم',\n",
              "  'و',\n",
              "  'از',\n",
              "  'خنده',\n",
              "  'بیهوش',\n",
              "  'شدم',\n",
              "  '.'],\n",
              " ['اصلا',\n",
              "  'درک',\n",
              "  'نمیکنم',\n",
              "  'که',\n",
              "  'چرا',\n",
              "  'تو',\n",
              "  'رقابت',\n",
              "  'دوست',\n",
              "  'داریم',\n",
              "  'طرف',\n",
              "  'نابود',\n",
              "  'بشه',\n",
              "  '!',\n",
              "  '؟',\n",
              "  'خب',\n",
              "  'به',\n",
              "  'پیروزی',\n",
              "  'قانع',\n",
              "  'باشیم',\n",
              "  'دیگه',\n",
              "  '!',\n",
              "  'حتما',\n",
              "  'که',\n",
              "  'نباید',\n",
              "  'نابودش',\n",
              "  'کنیم',\n",
              "  'که',\n",
              "  '!'],\n",
              " ['توی',\n",
              "  'جریان',\n",
              "  'واکسن',\n",
              "  'آنفولانزا',\n",
              "  'و',\n",
              "  'حاشیه\\u200cهای',\n",
              "  'قبل',\n",
              "  'از',\n",
              "  'این',\n",
              "  'بنظرم',\n",
              "  'از',\n",
              "  'بی\\u200cتجربگی',\n",
              "  'نماینده\\u200cها',\n",
              "  'استفاده',\n",
              "  'کرد',\n",
              "  'و',\n",
              "  'فشار',\n",
              "  'بهشون',\n",
              "  'تحمیل',\n",
              "  'کرد',\n",
              "  '.'],\n",
              " ['خود',\n",
              "  'معجزه',\n",
              "  'هزاره',\n",
              "  'هم',\n",
              "  'محصول',\n",
              "  'و',\n",
              "  'برکشیده',\n",
              "  'دولت',\n",
              "  'پنهان',\n",
              "  'بود',\n",
              "  'که',\n",
              "  'با',\n",
              "  'انکار',\n",
              "  'آن',\n",
              "  'و',\n",
              "  'توهم',\n",
              "  'پایگاه',\n",
              "  'و',\n",
              "  'رای\\u200cآوری',\n",
              "  'خودشمنحرف',\n",
              "  'شد'],\n",
              " ['من',\n",
              "  'جوابی',\n",
              "  'در',\n",
              "  'دفاع',\n",
              "  'از',\n",
              "  'احزاب',\n",
              "  'نداشتم',\n",
              "  '،',\n",
              "  'اشاره',\n",
              "  'من',\n",
              "  'به',\n",
              "  'ناکارآمدی',\n",
              "  'احزاب',\n",
              "  'به',\n",
              "  'واسطه',\n",
              "  'سیستم',\n",
              "  'حاکمی',\n",
              "  'هست',\n",
              "  'چیزی',\n",
              "  'حز',\n",
              "  'یک',\n",
              "  'کاریکاتور',\n",
              "  'از',\n",
              "  'اونها',\n",
              "  'باقی',\n",
              "  'نگذاشتند',\n",
              "  '.'],\n",
              " ['من',\n",
              "  'هشت',\n",
              "  'ساله',\n",
              "  'از',\n",
              "  'جدا',\n",
              "  'از',\n",
              "  'پدر',\n",
              "  'دخترم',\n",
              "  'زندگی',\n",
              "  'میکنم',\n",
              "  'ولی',\n",
              "  'بخاطر',\n",
              "  'سنگ',\n",
              "  'اندازی\\u200cهایی',\n",
              "  'که',\n",
              "  'میکرد',\n",
              "  'کارهای',\n",
              "  'قانونی',\n",
              "  'طلاق',\n",
              "  'انجام',\n",
              "  'نمیشد',\n",
              "  '.',\n",
              "  'دوسال',\n",
              "  'هست',\n",
              "  'با',\n",
              "  'آقایی',\n",
              "  'آشنا',\n",
              "  'شدم',\n",
              "  'که',\n",
              "  'دیدم',\n",
              "  'رو',\n",
              "  'نسبت',\n",
              "  'به',\n",
              "  'مردها',\n",
              "  'عوض',\n",
              "  'کرده',\n",
              "  '.',\n",
              "  'از',\n",
              "  'پدر',\n",
              "  'دخترم',\n",
              "  'دارم',\n",
              "  'جدا',\n",
              "  'میشم',\n",
              "  'ولی',\n",
              "  'گل',\n",
              "  'هدیه',\n",
              "  'مردی',\n",
              "  'هست',\n",
              "  'که',\n",
              "  'دو',\n",
              "  'ساله',\n",
              "  'توی',\n",
              "  'زندگیمه'],\n",
              " ['ای',\n",
              "  'بابا',\n",
              "  'دیگه',\n",
              "  'اسم',\n",
              "  'دوست',\n",
              "  'پسرو',\n",
              "  'نیار',\n",
              "  'الان',\n",
              "  'غیرتی',\n",
              "  'میشم',\n",
              "  'همه',\n",
              "  'رو',\n",
              "  'به',\n",
              "  'رگبار',\n",
              "  'میگیرم',\n",
              "  'من',\n",
              "  'هستم',\n",
              "  'دیگه'],\n",
              " ['حال',\n",
              "  'خوب',\n",
              "  'رو',\n",
              "  'بهمون',\n",
              "  'نمیدن',\n",
              "  '.',\n",
              "  'حال',\n",
              "  'خوب',\n",
              "  'را',\n",
              "  'باید',\n",
              "  'ساخت',\n",
              "  '.',\n",
              "  'با',\n",
              "  'تکه',\n",
              "  'نان',\n",
              "  'سنگگی',\n",
              "  'و',\n",
              "  'مقداری',\n",
              "  'عسل',\n",
              "  'و',\n",
              "  'سرشیر',\n",
              "  'و',\n",
              "  'چای',\n",
              "  'گرم',\n",
              "  'و',\n",
              "  'با',\n",
              "  'شنیدن',\n",
              "  'صدای',\n",
              "  'دلنشین',\n",
              "  'بنان',\n",
              "  '.',\n",
              "  'صبح',\n",
              "  'بخیر',\n",
              "  'زندگی'],\n",
              " ['هدف',\n",
              "  'کشته\\u200cسازی',\n",
              "  'نبوده',\n",
              "  'ولی',\n",
              "  'نه',\n",
              "  'اینکه',\n",
              "  'تلفات',\n",
              "  'نداشتند',\n",
              "  '.',\n",
              "  'همون',\n",
              "  'موقع',\n",
              "  'هم',\n",
              "  'اعلام',\n",
              "  'کردند',\n",
              "  'اگر',\n",
              "  'هدف',\n",
              "  'کشته',\n",
              "  'سازی',\n",
              "  'بود',\n",
              "  'استراحتگاه',\n",
              "  'سربازان',\n",
              "  'و',\n",
              "  'می\\u200cزدند',\n",
              "  '.',\n",
              "  'این',\n",
              "  'هم',\n",
              "  'مدرک',\n",
              "  'تلفات',\n",
              "  'امریکائی\\u200cها',\n",
              "  'که',\n",
              "  'توسط',\n",
              "  'خودشون',\n",
              "  'اعلام',\n",
              "  'شد'],\n",
              " ['آقا',\n",
              "  'جان',\n",
              "  'کی',\n",
              "  'گفته',\n",
              "  'درمان',\n",
              "  'نمی\\u200cکرده',\n",
              "  '.',\n",
              "  'اساسا',\n",
              "  'بدن',\n",
              "  'انسان',\n",
              "  'خودش',\n",
              "  'کلی',\n",
              "  'از',\n",
              "  'بیماری\\u200cها',\n",
              "  'را',\n",
              "  'درمان',\n",
              "  'می\\u200cکنه',\n",
              "  'و',\n",
              "  'طب',\n",
              "  'سنتی',\n",
              "  'لااقل',\n",
              "  'قوای',\n",
              "  'بدن',\n",
              "  'را',\n",
              "  'تقویت',\n",
              "  'که',\n",
              "  'می\\u200cکنه',\n",
              "  '.',\n",
              "  'مسئله',\n",
              "  'اثربخشی',\n",
              "  'روش\\u200cها',\n",
              "  'ست',\n",
              "  '.',\n",
              "  'و',\n",
              "  'مسئله',\n",
              "  'دیگر',\n",
              "  'اینکه',\n",
              "  'یه',\n",
              "  'عده',\n",
              "  'به',\n",
              "  'نام',\n",
              "  'طب',\n",
              "  'سنتی',\n",
              "  'کلاشی',\n",
              "  'و',\n",
              "  'بازی',\n",
              "  'رسانه\\u200cای',\n",
              "  'می\\u200cکنند',\n",
              "  'تا',\n",
              "  'مردم',\n",
              "  'را',\n",
              "  'بچاپند'],\n",
              " ['شدنش',\n",
              "  'میشه',\n",
              "  'ولی',\n",
              "  'برد',\n",
              "  'بلند',\n",
              "  'معمولا',\n",
              "  'از',\n",
              "  'زیرسطحی',\n",
              "  'پرتاب',\n",
              "  'میکنن',\n",
              "  '،',\n",
              "  'کلاس',\n",
              "  'فاتح',\n",
              "  'و',\n",
              "  'رعد',\n",
              "  'میشه',\n",
              "  'عمود',\n",
              "  'پرتاب',\n",
              "  'کرد',\n",
              "  'و',\n",
              "  'از',\n",
              "  'روی',\n",
              "  'ناو',\n",
              "  'هم',\n",
              "  'شلیک',\n",
              "  'کرد'],\n",
              " ['کیک',\n",
              "  'داشته',\n",
              "  'باشه',\n",
              "  'یعنی',\n",
              "  'یکم',\n",
              "  'تاثیر',\n",
              "  'داشته',\n",
              "  'باشه',\n",
              "  'روم',\n",
              "  '.',\n",
              "  'بترسونم',\n",
              "  '.'],\n",
              " ['دست', 'ما', 'رو', 'قطع', 'نکنی', 'بچه', 'شمرون'],\n",
              " ['.',\n",
              "  'آقای',\n",
              "  'عزیز',\n",
              "  '-',\n",
              "  'نوبت',\n",
              "  'را',\n",
              "  'رعایت',\n",
              "  'کنید',\n",
              "  'دفه',\n",
              "  'دیگر',\n",
              "  '.',\n",
              "  'شما',\n",
              "  'خیلی',\n",
              "  'از',\n",
              "  'گفته\\u200cهاتون',\n",
              "  'دروست',\n",
              "  'نیست',\n",
              "  '.',\n",
              "  'و',\n",
              "  'دانش',\n",
              "  'ان',\n",
              "  'را',\n",
              "  'ندارد',\n",
              "  '-',\n",
              "  'این',\n",
              "  'بی\\u200cادبی',\n",
              "  'هم',\n",
              "  'هاست',\n",
              "  'که',\n",
              "  'بخندید',\n",
              "  'به',\n",
              "  'گفته\\u200cهای',\n",
              "  'دیگران',\n",
              "  '-',\n",
              "  'ما',\n",
              "  'هم',\n",
              "  'از',\n",
              "  'تو',\n",
              "  'باسوادتر',\n",
              "  'هستیم',\n",
              "  'و',\n",
              "  'هم',\n",
              "  'در',\n",
              "  'یک',\n",
              "  'محیط',\n",
              "  'آزاد',\n",
              "  'بزرگ',\n",
              "  'شدیم',\n",
              "  '،',\n",
              "  'موااداب',\n",
              "  'باش',\n",
              "  'یا',\n",
              "  'دیگه',\n",
              "  'نیا',\n",
              "  'حرف\\u200cها',\n",
              "  'ی',\n",
              "  'مفت',\n",
              "  'بزنی'],\n",
              " ['تنها',\n",
              "  'کسی',\n",
              "  'که',\n",
              "  'بعد',\n",
              "  '15',\n",
              "  'سال',\n",
              "  'هنوز',\n",
              "  'شناخت',\n",
              "  'افکارش',\n",
              "  'توسط',\n",
              "  'مردم',\n",
              "  'لرزه',\n",
              "  'به',\n",
              "  'تن',\n",
              "  'الگاریشی',\n",
              "  'مینداره',\n",
              "  '!'],\n",
              " ['تنها',\n",
              "  'جایی',\n",
              "  'که',\n",
              "  'در',\n",
              "  'امان',\n",
              "  'هستی',\n",
              "  'تا',\n",
              "  'یکی',\n",
              "  'بهت',\n",
              "  'نگه',\n",
              "  '«',\n",
              "  'اینقدر',\n",
              "  'میری',\n",
              "  'سر',\n",
              "  'موبایلت',\n",
              "  '»',\n",
              "  'کجاست',\n",
              "  '؟',\n",
              "  '.',\n",
              "  'من',\n",
              "  ':',\n",
              "  'دستشویی',\n",
              "  'فرنگی'],\n",
              " ['موهای',\n",
              "  'منم',\n",
              "  'پشمیه',\n",
              "  '،',\n",
              "  'نه',\n",
              "  'نگهداری',\n",
              "  'میکنم',\n",
              "  'که',\n",
              "  'فرش',\n",
              "  'قشنگ',\n",
              "  'بشه',\n",
              "  '،',\n",
              "  'نه',\n",
              "  'حال',\n",
              "  'دارم',\n",
              "  'صافش',\n",
              "  'کنم',\n",
              "  ':))))'],\n",
              " ['داریم',\n",
              "  'وارد',\n",
              "  'لایه\\u200cهای',\n",
              "  'عمیق\\u200cتری',\n",
              "  'از',\n",
              "  'ماتریکس',\n",
              "  'می\\u200cشیم'],\n",
              " ['اینا',\n",
              "  'با',\n",
              "  'بازی',\n",
              "  'ساختن',\n",
              "  'میخوان',\n",
              "  'مردم',\n",
              "  'رو',\n",
              "  'بترسونن',\n",
              "  'و',\n",
              "  'خودشون',\n",
              "  'هم',\n",
              "  'از',\n",
              "  'بازی',\n",
              "  'می\\u200cترسن',\n",
              "  '.',\n",
              "  'واقعا',\n",
              "  'چقدر',\n",
              "  'ضایع'],\n",
              " ['6', 'ماه', 'شب', 'شفقای', 'قطبی', 'غرق', 'شدن', 'ته', 'اقیانوس', '.'],\n",
              " ['این',\n",
              "  'پنیر',\n",
              "  'موزرلا',\n",
              "  'بوراتا',\n",
              "  '،',\n",
              "  'دقیقا',\n",
              "  'مزه',\n",
              "  'سرشیر',\n",
              "  'میده',\n",
              "  '،',\n",
              "  'من',\n",
              "  'میگیرم',\n",
              "  'و',\n",
              "  'روش',\n",
              "  'عسل',\n",
              "  'میریزم',\n",
              "  'و',\n",
              "  'میخورم',\n",
              "  '،',\n",
              "  'جای',\n",
              "  'سرشیر',\n",
              "  'میشه',\n",
              "  'استفاده'],\n",
              " ['یک',\n",
              "  'کم',\n",
              "  'عجیبه',\n",
              "  'ولی',\n",
              "  '.',\n",
              "  'واقعا',\n",
              "  'این',\n",
              "  'حجم',\n",
              "  'تولید',\n",
              "  'سنتی',\n",
              "  'کشک',\n",
              "  'داریم',\n",
              "  '؟'],\n",
              " ['فقط',\n",
              "  'یادت',\n",
              "  'باشه',\n",
              "  'انفجار',\n",
              "  'مرکز',\n",
              "  'هسته\\u200cای',\n",
              "  'و',\n",
              "  'رآکتور',\n",
              "  'قارچ',\n",
              "  'نداره',\n",
              "  '،',\n",
              "  'ولی',\n",
              "  'تشعشع',\n",
              "  'داره'],\n",
              " ['صبح\\u200cهای',\n",
              "  'تنگ',\n",
              "  '،',\n",
              "  'افتاب',\n",
              "  'بی\\u200cجان',\n",
              "  '،',\n",
              "  'کثیفی',\n",
              "  'یک',\n",
              "  'پسر',\n",
              "  'بچه',\n",
              "  '،',\n",
              "  'کثافت',\n",
              "  'درون',\n",
              "  'شهر',\n",
              "  '.',\n",
              "  'خواب\\u200cهای',\n",
              "  'عمیق',\n",
              "  'بزرگ\\u200cها',\n",
              "  '.',\n",
              "  'خوب',\n",
              "  'است',\n",
              "  'که',\n",
              "  'بیدارم',\n",
              "  '!',\n",
              "  '؟',\n",
              "  'نمیدانم',\n",
              "  '.',\n",
              "  'قضیه',\n",
              "  'این',\n",
              "  'نیست',\n",
              "  '،',\n",
              "  'قضیه',\n",
              "  'رابطه',\n",
              "  'نامشروع',\n",
              "  'بین',\n",
              "  'ثانیه\\u200cهای',\n",
              "  'نجس',\n",
              "  'است',\n",
              "  'که',\n",
              "  'بین',\n",
              "  'دست\\u200cها',\n",
              "  'و',\n",
              "  'یک',\n",
              "  'به',\n",
              "  'صلاح',\n",
              "  'حال',\n",
              "  'خوب',\n",
              "  'کن',\n",
              "  '.',\n",
              "  'در',\n",
              "  'گردش',\n",
              "  'است',\n",
              "  '.'],\n",
              " ['گل', 'در', 'اندیشه', 'که', 'کی', 'خودرو', 'کند', 'در', 'پاچه\\u200cش', '!'],\n",
              " ['عزیزم',\n",
              "  'منم',\n",
              "  'خوندم',\n",
              "  '.',\n",
              "  'اما',\n",
              "  'توضیحش',\n",
              "  'این',\n",
              "  'بود',\n",
              "  'که',\n",
              "  'این',\n",
              "  'اتفاق',\n",
              "  'معمولا',\n",
              "  'توی',\n",
              "  'استخرهای',\n",
              "  'شخصی',\n",
              "  'که',\n",
              "  'کلر',\n",
              "  'کمتر',\n",
              "  'داره',\n",
              "  'و',\n",
              "  'دمای',\n",
              "  'آب',\n",
              "  'به',\n",
              "  'دمای',\n",
              "  'بدن',\n",
              "  'نزدیک',\n",
              "  'هست',\n",
              "  'میفته',\n",
              "  'نه',\n",
              "  'استخرهای',\n",
              "  'عمومی',\n",
              "  'که',\n",
              "  'از',\n",
              "  'بوی',\n",
              "  'کلر',\n",
              "  'نمیتونی',\n",
              "  'نفس',\n",
              "  'بکشی',\n",
              "  ':)))'],\n",
              " ['آیا',\n",
              "  'برایتان',\n",
              "  'سوال',\n",
              "  'نشده',\n",
              "  'چرا',\n",
              "  'پس',\n",
              "  'از',\n",
              "  'این',\n",
              "  'تجمعات',\n",
              "  'پیگیری',\n",
              "  'دیگری',\n",
              "  'برای',\n",
              "  'تحقق',\n",
              "  'این',\n",
              "  'مطالبات',\n",
              "  'طراحی',\n",
              "  'شده\\u200c',\n",
              "  '!',\n",
              "  'صورت',\n",
              "  'نمیگیرد',\n",
              "  'ولی',\n",
              "  'در',\n",
              "  'کمتر',\n",
              "  'از',\n",
              "  'یک',\n",
              "  'هفته',\n",
              "  'عملیات',\n",
              "  'دستگیری',\n",
              "  'موجی',\n",
              "  'از',\n",
              "  'معترضان',\n",
              "  'در',\n",
              "  'آن',\n",
              "  'تجمع',\n",
              "  'شروع',\n",
              "  'می\\u200cشود',\n",
              "  '؟'],\n",
              " ['از', 'صمیم', 'قلب', 'ممنون', 'سیا', 'چومه', 'جان'],\n",
              " ['شاید', 'همشون', 'ایدز', 'دارند', '!'],\n",
              " ['در',\n",
              "  'توئیتر',\n",
              "  'هم',\n",
              "  'این',\n",
              "  '\"',\n",
              "  'جنبش',\n",
              "  '\"',\n",
              "  'هست',\n",
              "  '؟',\n",
              "  'من',\n",
              "  'ندیدم',\n",
              "  '!'],\n",
              " ['فقط',\n",
              "  'برای',\n",
              "  'خودت',\n",
              "  'زندگی',\n",
              "  'کن',\n",
              "  'و',\n",
              "  'خوشی',\n",
              "  'زندگیت',\n",
              "  'رو',\n",
              "  'به',\n",
              "  'هیچکس',\n",
              "  'پیوند',\n",
              "  'نزن'],\n",
              " ['تاکنون',\n",
              "  'سایر',\n",
              "  'بلوک\\u200cهای',\n",
              "  'مهم',\n",
              "  'اولیه',\n",
              "  'زمینه\\u200cساز',\n",
              "  'حیات',\n",
              "  'در',\n",
              "  'شهاب',\n",
              "  'سنگ\\u200cها',\n",
              "  'از',\n",
              "  'جمله',\n",
              "  'اسیدهای',\n",
              "  'آمینه',\n",
              "  '(',\n",
              "  'اجزای',\n",
              "  'پروتئین',\n",
              "  ')',\n",
              "  'و',\n",
              "  'باز',\n",
              "  'نوکلئوتیدی',\n",
              "  '(',\n",
              "  'اجزای',\n",
              "  'DNA',\n",
              "  'و',\n",
              "  'RNA',\n",
              "  ')',\n",
              "  'یافت',\n",
              "  'شده',\n",
              "  'است',\n",
              "  '،',\n",
              "  'اما',\n",
              "  'قندها',\n",
              "  'در',\n",
              "  'میان',\n",
              "  'بخش\\u200cهای',\n",
              "  'اصلی',\n",
              "  'زمینه',\n",
              "  'ساز',\n",
              "  'حیات',\n",
              "  '،',\n",
              "  'قطعه',\n",
              "  'گمشده',\n",
              "  'این',\n",
              "  'پازل',\n",
              "  'بودند',\n",
              "  '.',\n",
              "  'این',\n",
              "  'تحقیق',\n",
              "  'اولین',\n",
              "  'شواهد',\n",
              "  'مستقیم',\n",
              "  'از',\n",
              "  'وجود',\n",
              "  'ریبوز',\n",
              "  'در',\n",
              "  'فضا',\n",
              "  'است'],\n",
              " ['دوستان',\n",
              "  'این',\n",
              "  'تصویر',\n",
              "  'و',\n",
              "  'ریتویت',\n",
              "  'و',\n",
              "  'لایک',\n",
              "  'نکنید',\n",
              "  'چهرهای',\n",
              "  'بچه\\u200cها',\n",
              "  'مشخصه',\n",
              "  '،',\n",
              "  'اصلا',\n",
              "  'درست',\n",
              "  'نیست',\n",
              "  'کودکان',\n",
              "  'را',\n",
              "  'وارد',\n",
              "  'نزاع',\n",
              "  'و',\n",
              "  'جنگ',\n",
              "  'سیاسی',\n",
              "  'کرد'],\n",
              " ['تو',\n",
              "  'این',\n",
              "  'کتفرانسه',\n",
              "  'بعد',\n",
              "  'از',\n",
              "  'چندین',\n",
              "  'سمینار',\n",
              "  'فیزیک',\n",
              "  'و',\n",
              "  'بیولوژی',\n",
              "  'خیلی',\n",
              "  'یهویی',\n",
              "  'یه',\n",
              "  'مربی',\n",
              "  'یوگا',\n",
              "  'آوردن',\n",
              "  'که',\n",
              "  'در',\n",
              "  'مورد',\n",
              "  'سفر',\n",
              "  'در',\n",
              "  'زمان',\n",
              "  'و',\n",
              "  'تجربیات',\n",
              "  'خودش',\n",
              "  'در',\n",
              "  'این',\n",
              "  'زمینه',\n",
              "  'و',\n",
              "  'این',\n",
              "  'که',\n",
              "  'چطور',\n",
              "  'باعث',\n",
              "  'رشد',\n",
              "  'شخصیتش',\n",
              "  'شده',\n",
              "  'صحبت',\n",
              "  'کرد',\n",
              "  '.'],\n",
              " ['خواهرمو',\n",
              "  'آوردم',\n",
              "  'واسه',\n",
              "  'کنکور',\n",
              "  'عملی',\n",
              "  '.',\n",
              "  'این',\n",
              "  'همه',\n",
              "  'جمعیت',\n",
              "  'آدم',\n",
              "  'و',\n",
              "  'یه',\n",
              "  'نفر',\n",
              "  'با',\n",
              "  'دماسنج',\n",
              "  'دم',\n",
              "  'در',\n",
              "  'ورودی',\n",
              "  '.',\n",
              "  'قشنگ',\n",
              "  'سیرکه',\n",
              "  '.'],\n",
              " ['من',\n",
              "  'متولد',\n",
              "  '1341',\n",
              "  'هستم',\n",
              "  '.',\n",
              "  'از',\n",
              "  'تمامی',\n",
              "  'دبیران',\n",
              "  'راهنمایی',\n",
              "  'و',\n",
              "  'دبیرستان',\n",
              "  'هم',\n",
              "  'یادمه',\n",
              "  'و',\n",
              "  'به',\n",
              "  'دیدنشون',\n",
              "  'میرم',\n",
              "  '،'],\n",
              " ['اسپری',\n",
              "  'و',\n",
              "  'خالی',\n",
              "  'کردم',\n",
              "  'تو',\n",
              "  'طبقه',\n",
              "  'بالا',\n",
              "  '.',\n",
              "  'کلا',\n",
              "  'بوش',\n",
              "  'بالا',\n",
              "  'بیشتر',\n",
              "  'از',\n",
              "  'خود',\n",
              "  'اشپزخونه',\n",
              "  'بود',\n",
              "  '.',\n",
              "  'واقعااا',\n",
              "  'درک',\n",
              "  'نمیکنم',\n",
              "  'چرا',\n",
              "  'ساعت',\n",
              "  '10',\n",
              "  'شب',\n",
              "  'باید',\n",
              "  'پیاز',\n",
              "  'سرخ',\n",
              "  'بشه'],\n",
              " ['تا',\n",
              "  'یه',\n",
              "  'جایی',\n",
              "  'تاثیرش',\n",
              "  'میتونه',\n",
              "  'مثبت',\n",
              "  'هم',\n",
              "  'باشه',\n",
              "  '،',\n",
              "  'اما',\n",
              "  'مصرف',\n",
              "  'زیادش',\n",
              "  'حافظه',\n",
              "  'و',\n",
              "  'تصمیم\\u200cگیری',\n",
              "  'رو',\n",
              "  'تحت',\n",
              "  'تاثیر',\n",
              "  'منفی',\n",
              "  'میذاره',\n",
              "  '.',\n",
              "  'بر',\n",
              "  'اساس',\n",
              "  'تجربه',\n",
              "  'خودم',\n",
              "  'و',\n",
              "  'اطرافیانم',\n",
              "  '.'],\n",
              " ['عرض',\n",
              "  'کنم',\n",
              "  'به',\n",
              "  'عنوان',\n",
              "  'یک',\n",
              "  'آدم',\n",
              "  'غیرمتخصص',\n",
              "  'پاسخ',\n",
              "  'می\\u200cدهم',\n",
              "  '،',\n",
              "  'تا',\n",
              "  'جایی',\n",
              "  'که',\n",
              "  'می\\u200cدانم',\n",
              "  'فدرالیسم',\n",
              "  'به',\n",
              "  'لحاظ',\n",
              "  'تاریخی',\n",
              "  'برعکس',\n",
              "  'این',\n",
              "  'راه',\n",
              "  'را',\n",
              "  'طی',\n",
              "  'می\\u200cکند',\n",
              "  'و',\n",
              "  'واحدهای',\n",
              "  'قومی',\n",
              "  '(',\n",
              "  'ملی',\n",
              "  '؟',\n",
              "  ')',\n",
              "  'توافقی',\n",
              "  'در',\n",
              "  'قالب',\n",
              "  'یک',\n",
              "  'واحد',\n",
              "  'سیاسی',\n",
              "  'مشترک',\n",
              "  'در',\n",
              "  'می\\u200cآیند',\n",
              "  '،',\n",
              "  'حالا',\n",
              "  'در',\n",
              "  'مسیر',\n",
              "  'عکس',\n",
              "  'یک',\n",
              "  'واحد',\n",
              "  'فعلا',\n",
              "  'یکپارچه',\n",
              "  'را',\n",
              "  'یک',\n",
              "  'قدم',\n",
              "  'به',\n",
              "  'سمت',\n",
              "  '\"',\n",
              "  'استقلال',\n",
              "  '\"',\n",
              "  'نزدیک',\n",
              "  'کنیم',\n",
              "  'بعید',\n",
              "  'است',\n",
              "  'کمکی',\n",
              "  'کند',\n",
              "  '.']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kq-WkFm4PyuC",
        "outputId": "06401f66-8307-45d5-bafc-18fcb23ea587"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "model = TFAutoModelForTokenClassification.from_pretrained(\"overfit/twiner-bert-base-mtl\")\n",
        "twiner_mtl = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 13:50:51 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:52 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 13:50:52 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:52 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-04-18 13:50:52 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:53 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-04-18 13:50:53 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:53 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-04-18 13:50:53 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:54 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-04-18 13:50:54 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:55 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 13:50:55 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:56 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-04-18 13:50:56 urllib3.connectionpool DEBUG: Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-04-18 13:50:56 urllib3.connectionpool DEBUG: https://huggingface.co:443 \"HEAD /overfit/twiner-bert-base-mtl/resolve/main/tf_model.h5 HTTP/1.1\" 302 0\n",
            "Some layers from the model checkpoint at overfit/twiner-bert-base-mtl were not used when initializing TFBertForTokenClassification: ['dropout_37']\n",
            "- This IS expected if you are initializing TFBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at overfit/twiner-bert-base-mtl.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dNEBzlCDP4to",
        "outputId": "9a48de13-5ffb-4270-d7e5-7c873903cdb7"
      },
      "source": [
        "tag_map = {\n",
        "  \"B-EVE\": \"B-EVE\",\n",
        "  \"B-LOC\": \"B-LOC\",\n",
        "  \"B-ORG\": \"B-ORG\",\n",
        "  \"B-PER\": \"B-PER\",\n",
        "  \"B-POG\": \"B-POG\",\n",
        "  \"B-NAT\": \"B-NAT\",\n",
        "  \"I-EVE\": \"I-EVE\",\n",
        "  \"I-LOC\": \"I-LOC\",\n",
        "  \"I-ORG\": \"I-ORG\",\n",
        "  \"I-PER\": \"I-PER\",\n",
        "  \"I-POG\": \"I_POG\",\n",
        "  \"I-NAT\": \"I-NAT\",\n",
        "  'O': 'O'\n",
        "}\n",
        "y_true, y_perd = align_prediction(diff_tweet, diff_tags, twiner_mtl, tag_map)\n",
        "benchmark(y_true, y_perd, ['PER', 'LOC', 'ORG'])"
      ],
      "execution_count": 140,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-18 14:51:13 root INFO: Imported 1 predictions for 1 true examples\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ent_type': {'actual': 15,\n",
              "   'correct': 5,\n",
              "   'incorrect': 1,\n",
              "   'missed': 2,\n",
              "   'partial': 0,\n",
              "   'possible': 8,\n",
              "   'precision': 0.3333333333333333,\n",
              "   'recall': 0.625,\n",
              "   'spurious': 9},\n",
              "  'exact': {'actual': 15,\n",
              "   'correct': 6,\n",
              "   'incorrect': 0,\n",
              "   'missed': 2,\n",
              "   'partial': 0,\n",
              "   'possible': 8,\n",
              "   'precision': 0.4,\n",
              "   'recall': 0.75,\n",
              "   'spurious': 9},\n",
              "  'partial': {'actual': 15,\n",
              "   'correct': 6,\n",
              "   'incorrect': 0,\n",
              "   'missed': 2,\n",
              "   'partial': 0,\n",
              "   'possible': 8,\n",
              "   'precision': 0.4,\n",
              "   'recall': 0.75,\n",
              "   'spurious': 9},\n",
              "  'strict': {'actual': 15,\n",
              "   'correct': 5,\n",
              "   'incorrect': 1,\n",
              "   'missed': 2,\n",
              "   'partial': 0,\n",
              "   'possible': 8,\n",
              "   'precision': 0.3333333333333333,\n",
              "   'recall': 0.625,\n",
              "   'spurious': 9}},\n",
              " {'LOC': {'ent_type': {'actual': 4,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.75,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'exact': {'actual': 4,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.75,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'partial': {'actual': 4,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.75,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1},\n",
              "   'strict': {'actual': 4,\n",
              "    'correct': 3,\n",
              "    'incorrect': 0,\n",
              "    'missed': 0,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.75,\n",
              "    'recall': 1.0,\n",
              "    'spurious': 1}},\n",
              "  'ORG': {'ent_type': {'actual': 5,\n",
              "    'correct': 0,\n",
              "    'incorrect': 1,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 4},\n",
              "   'exact': {'actual': 5,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.5,\n",
              "    'spurious': 4},\n",
              "   'partial': {'actual': 5,\n",
              "    'correct': 1,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0.2,\n",
              "    'recall': 0.5,\n",
              "    'spurious': 4},\n",
              "   'strict': {'actual': 5,\n",
              "    'correct': 0,\n",
              "    'incorrect': 1,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 2,\n",
              "    'precision': 0.0,\n",
              "    'recall': 0.0,\n",
              "    'spurious': 4}},\n",
              "  'PER': {'ent_type': {'actual': 6,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.3333333333333333,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 4},\n",
              "   'exact': {'actual': 6,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.3333333333333333,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 4},\n",
              "   'partial': {'actual': 6,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.3333333333333333,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 4},\n",
              "   'strict': {'actual': 6,\n",
              "    'correct': 2,\n",
              "    'incorrect': 0,\n",
              "    'missed': 1,\n",
              "    'partial': 0,\n",
              "    'possible': 3,\n",
              "    'precision': 0.3333333333333333,\n",
              "    'recall': 0.6666666666666666,\n",
              "    'spurious': 4}}})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-lQw00CjRizL"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"HooshvareLab/bert-base-parsbert-peymaner-uncased\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"HooshvareLab/bert-base-parsbert-peymaner-uncased\")\n",
        "model.eval()\n",
        "parsbert_peyma = pipeline('ner', model=model, tokenizer=tokenizer, ignore_labels=[])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mUdOaKFgQSUc"
      },
      "source": [
        "#### Peyma"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Y1P9ggAQA_Z"
      },
      "source": [
        "testset_named_entities = []\n",
        "for tweet, tweet_tags in zip(texts_peyma, tags_peyma):\n",
        "  for word, tag in zip(tweet, tweet_tags):\n",
        "    if tag != 'O':\n",
        "      testset_named_entities.append(word)"
      ],
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HYcT7XraQpVX"
      },
      "source": [
        "trainset_named_entities = []\n",
        "for tweet, tweet_tags in zip(texts_peyma_train, tags_peyma_train):\n",
        "  for word, tag in zip(tweet, tweet_tags):\n",
        "    if tag != 'O':\n",
        "      trainset_named_entities.append(word)"
      ],
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_Mg0xG0Qvop"
      },
      "source": [
        "diff_test_train = set(testset_named_entities) - set(trainset_named_entities)"
      ],
      "execution_count": 115,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oInTDbgSQ0N_",
        "outputId": "c476b764-eb47-485f-c5ff-10b386bcd153"
      },
      "source": [
        "len(diff_test_train)"
      ],
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aZnuFcFIQ15W"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}