Pre_trained_encoder_neural_machine_translation_with_transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "Pre-trained encoder neural_machine_translation_with_transformer",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "940f7939964a42c6ad2a82d6ae3b091f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7aecd392fe514e22b024b0f6ea4e8eae",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d3ee2e58ecd546779572760996857c1d",
              "IPY_MODEL_357d66b3c39845e7b91aea7e9df6c7c6",
              "IPY_MODEL_1751801ea9d044fea02f9da9c28db1e3"
            ]
          }
        },
        "7aecd392fe514e22b024b0f6ea4e8eae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d3ee2e58ecd546779572760996857c1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a062984d177645bc98069b2e025f4d39",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9a95637be3f1464281a1aefa70a34f8e"
          }
        },
        "357d66b3c39845e7b91aea7e9df6c7c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ce9fb2f1856a4a38b964b45fe749997b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 791656,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791656,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c82c009a6824a44bec384fc0cb1797c"
          }
        },
        "1751801ea9d044fea02f9da9c28db1e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b9c8fe2aa88642cfa3b93bb05449a24d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 773k/773k [00:00&lt;00:00, 1.37MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b84031172f2544e08fc3b937378f55da"
          }
        },
        "a062984d177645bc98069b2e025f4d39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9a95637be3f1464281a1aefa70a34f8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce9fb2f1856a4a38b964b45fe749997b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9c82c009a6824a44bec384fc0cb1797c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9c8fe2aa88642cfa3b93bb05449a24d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b84031172f2544e08fc3b937378f55da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c3b1a3aecfe47b5aa93f34dbd757e8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e83a91d035f64dc085de5f0b1ffbc48d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e29db46551d54469bfbad591d8492e7d",
              "IPY_MODEL_eb0d9a7715fd4b90bc08c54c72c344b6",
              "IPY_MODEL_e73a88c265b7433288e30c8909818828"
            ]
          }
        },
        "e83a91d035f64dc085de5f0b1ffbc48d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e29db46551d54469bfbad591d8492e7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c1cfc00cf0e847629dfc94e1b3ab70c7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fb7776b48bb54ec88ac4e76a2a0f0ac8"
          }
        },
        "eb0d9a7715fd4b90bc08c54c72c344b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_887cc78501424f5bbd0146d5824595a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1786,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1786,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f2eac8eab2db48e49b191ef9fdc64485"
          }
        },
        "e73a88c265b7433288e30c8909818828": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fdfe94cd6aef4f4e8857b25b1422e4ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.74k/1.74k [00:00&lt;00:00, 29.2kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d64253b27bfe446dba1873ef68017666"
          }
        },
        "c1cfc00cf0e847629dfc94e1b3ab70c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fb7776b48bb54ec88ac4e76a2a0f0ac8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "887cc78501424f5bbd0146d5824595a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f2eac8eab2db48e49b191ef9fdc64485": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fdfe94cd6aef4f4e8857b25b1422e4ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d64253b27bfe446dba1873ef68017666": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8985decb7b9042b7ab17fd438c77a7a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_31ff1e662b1e426a9f475299e0606165",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_899c536e90394cc291e9b71024ff67da",
              "IPY_MODEL_4f3288ad853e42e59c04ad9590b78bb4",
              "IPY_MODEL_63f3063734c040a1a151676eedbee6a5"
            ]
          }
        },
        "31ff1e662b1e426a9f475299e0606165": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "899c536e90394cc291e9b71024ff67da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d769cc15ce3e4364b04b125a59f201b8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_99e973ad66b547538987881c28591324"
          }
        },
        "4f3288ad853e42e59c04ad9590b78bb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b85016da309542cc8c39b7c588b5306d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1857,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1857,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f0410fe0c95844738760e5da327a9b63"
          }
        },
        "63f3063734c040a1a151676eedbee6a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5d9393c00a794be4aa74e676117823ae",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.81k/1.81k [00:00&lt;00:00, 41.7kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07d513487e1c4fb39d661eb8bf1415e9"
          }
        },
        "d769cc15ce3e4364b04b125a59f201b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "99e973ad66b547538987881c28591324": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b85016da309542cc8c39b7c588b5306d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f0410fe0c95844738760e5da327a9b63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d9393c00a794be4aa74e676117823ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07d513487e1c4fb39d661eb8bf1415e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "25637fbda4ce4c65bf96b21f461149b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5b3eac0bc02e497296c72668857779fe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6a33798ccf2544ba89c9147db20e84e9",
              "IPY_MODEL_6b6b965a6a1d4c84abae85cd60b419da",
              "IPY_MODEL_bb279bc90fca4b198df83f182a46ba4c"
            ]
          }
        },
        "5b3eac0bc02e497296c72668857779fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6a33798ccf2544ba89c9147db20e84e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bf35fbde55c9468c8bec2bf0bb1859da",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_27012e6998944073abb2594cda29381c"
          }
        },
        "6b6b965a6a1d4c84abae85cd60b419da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_43986ec3ea8848feb3b49298aaf77699",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 537,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 537,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e940f47c371456da12e996bd2efcbe5"
          }
        },
        "bb279bc90fca4b198df83f182a46ba4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9cd345b312c9491fa0de0e6cabc7d00a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 537/537 [00:00&lt;00:00, 14.5kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d79a6cf81f141079724611c0afd83f8"
          }
        },
        "bf35fbde55c9468c8bec2bf0bb1859da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "27012e6998944073abb2594cda29381c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "43986ec3ea8848feb3b49298aaf77699": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e940f47c371456da12e996bd2efcbe5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9cd345b312c9491fa0de0e6cabc7d00a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d79a6cf81f141079724611c0afd83f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60514887103243f484c8215e7d8ed523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_438c5f356c9a4720a8e6f9dec4b76ef7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_56b84936f9a240ce96389b29696944d5",
              "IPY_MODEL_9a4ca384567e43dd9a7af2f99c53d19d",
              "IPY_MODEL_417241363a534a98a831d403ad002104"
            ]
          }
        },
        "438c5f356c9a4720a8e6f9dec4b76ef7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56b84936f9a240ce96389b29696944d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4fc27818b4534bdcb5c21522988b09a4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3707e92dc8f34c179c33d2aa665d4cf6"
          }
        },
        "9a4ca384567e43dd9a7af2f99c53d19d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c4d51f645eb8402eb5ad99c8879c415a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 235127,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 235127,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_985d3dc05f194dd99e329a338980242d"
          }
        },
        "417241363a534a98a831d403ad002104": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_35d48c41b0984e65875399a6eb1910a0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 230k/230k [00:00&lt;00:00, 1.23MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1818ae8b21964fdd9a9465bceca10acc"
          }
        },
        "4fc27818b4534bdcb5c21522988b09a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3707e92dc8f34c179c33d2aa665d4cf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4d51f645eb8402eb5ad99c8879c415a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "985d3dc05f194dd99e329a338980242d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "35d48c41b0984e65875399a6eb1910a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1818ae8b21964fdd9a9465bceca10acc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6a791b2f59bb42f6a04e159dbadf2292": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f4e7a05887f74aef8ea992bbf79a21d9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f76ee5d9a70c4fda870ca4d3fcc1b6d5",
              "IPY_MODEL_7fd62e8aa6524f1da5bd234ea61a88e5",
              "IPY_MODEL_345ec00a503240a5b0bdcc6923ee71a9"
            ]
          }
        },
        "f4e7a05887f74aef8ea992bbf79a21d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f76ee5d9a70c4fda870ca4d3fcc1b6d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_36b2de4e605d401ea0605383db4c2f0e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_db8d5d08ae33469f884343c8e9d0fde2"
          }
        },
        "7fd62e8aa6524f1da5bd234ea61a88e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_89981cc6d6c84dc9bf6cc6e9365bd427",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 59,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 59,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80daa77ab1794ba8b36ee2c8c1bfb988"
          }
        },
        "345ec00a503240a5b0bdcc6923ee71a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aa2e7a81ff5041ccb94bf7e4ab8aa28d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 59.0/59.0 [00:00&lt;00:00, 1.12kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ae0cf5818d64b97a1d04ae900a21436"
          }
        },
        "36b2de4e605d401ea0605383db4c2f0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "db8d5d08ae33469f884343c8e9d0fde2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "89981cc6d6c84dc9bf6cc6e9365bd427": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "80daa77ab1794ba8b36ee2c8c1bfb988": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa2e7a81ff5041ccb94bf7e4ab8aa28d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ae0cf5818d64b97a1d04ae900a21436": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "88c530eff74c481ebfa6a963b5148e8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b43a700a0af4467e8642801c88a15b3f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4693ba52d57f4206b9373ab8aabf4f9e",
              "IPY_MODEL_2a7751448cb34e7d87fadab5ca419324",
              "IPY_MODEL_dcb837cd796f4cb398061709663f74fa"
            ]
          }
        },
        "b43a700a0af4467e8642801c88a15b3f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4693ba52d57f4206b9373ab8aabf4f9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98d424e344574b2d9bea88db23e543fe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c365a69a49d546dd92c905f38e963dfd"
          }
        },
        "2a7751448cb34e7d87fadab5ca419324": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1aaa3503ada94f05af04e430920c8c13",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_54294d943c4a4a1aad5aab5e743a99b7"
          }
        },
        "dcb837cd796f4cb398061709663f74fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aab38013a26346bca32d2cd34a63690c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 9.23kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0cd1675a52a34ccb920d24d3b747c51a"
          }
        },
        "98d424e344574b2d9bea88db23e543fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c365a69a49d546dd92c905f38e963dfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1aaa3503ada94f05af04e430920c8c13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "54294d943c4a4a1aad5aab5e743a99b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aab38013a26346bca32d2cd34a63690c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0cd1675a52a34ccb920d24d3b747c51a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "327e0394f2e4479fa6c9019f9240382f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1bf3adfb4ff245708e8dba72795b5c96",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_57f332fd467e4c1ca2d21afbe7d4b643",
              "IPY_MODEL_87d9eec07ddc47bb91910b6838af90db",
              "IPY_MODEL_0207d92f40884ffab8ae4697a7109b78"
            ]
          }
        },
        "1bf3adfb4ff245708e8dba72795b5c96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57f332fd467e4c1ca2d21afbe7d4b643": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_07530cbe5ce448ca92d24a66d1a6d7d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a564baaeb4043bf882a7259df2dfe87"
          }
        },
        "87d9eec07ddc47bb91910b6838af90db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_33dc08585df546479426ac70de917e46",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 308231040,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 308231040,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cbf6f34d27db43bb86d372e123e5b5ed"
          }
        },
        "0207d92f40884ffab8ae4697a7109b78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4bf9db5d046b4a6ba3b01cc11b8d58a2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 294M/294M [00:09&lt;00:00, 31.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_39d0b6ce158048849dbd7de3ef9d6232"
          }
        },
        "07530cbe5ce448ca92d24a66d1a6d7d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a564baaeb4043bf882a7259df2dfe87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33dc08585df546479426ac70de917e46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cbf6f34d27db43bb86d372e123e5b5ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4bf9db5d046b4a6ba3b01cc11b8d58a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "39d0b6ce158048849dbd7de3ef9d6232": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AqocPvYesWWK"
      },
      "source": [
        "# English-to-Spanish translation with a sequence-to-sequence Transformer\n",
        "\n",
        "**Author:** [fchollet](https://twitter.com/fchollet)<br>\n",
        "**Date created:** 2021/05/26<br>\n",
        "**Last modified:** 2021/05/26<br>\n",
        "**Description:** Implementing a sequence-to-sequene Transformer and training it on a machine translation task."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ea9JQgjhsWWL"
      },
      "source": [
        "## Introduction\n",
        "\n",
        "In this example, we'll build a sequence-to-sequence Transformer model, which\n",
        "we'll train on an English-to-Spanish machine translation task.\n",
        "\n",
        "You'll learn how to:\n",
        "\n",
        "- Vectorize text using the Keras `TextVectorization` layer.\n",
        "- Implement a `TransformerEncoder` layer, a `TransformerDecoder` layer,\n",
        "and a `PositionalEmbedding` layer.\n",
        "- Prepare data for training a sequence-to-sequence model.\n",
        "- Use the trained model to generate translations of never-seen-before\n",
        "input sentences (sequence-to-sequence inference).\n",
        "\n",
        "The code featured here is adapted from the book\n",
        "[Deep Learning with Python, Second Edition](https://www.manning.com/books/deep-learning-with-python-second-edition)\n",
        "(chapter 11: Deep learning for text).\n",
        "The present example is fairly barebones, so for detailed explanations of\n",
        "how each building block works, as well as the theory behind Transformers,\n",
        "I recommend reading the book."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ztt9YLmZsWWM"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJFmFq2AplvT",
        "outputId": "596fdb0b-8022-45c5-81d0-74bb869c816d"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')  # mount the drive"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1eHtucw_fBpQ",
        "outputId": "95dfec02-580a-400f-f4e0-4cb889a424fb"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install sentencepiece\n",
        "!pip install datasets"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.12.5-py3-none-any.whl (3.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1 MB 5.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.2)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 45.1 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.1.2-py3-none-any.whl (59 kB)\n",
            "\u001b[K     |████████████████████████████████| 59 kB 6.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 38.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 45.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.1.2 pyyaml-6.0 sacremoses-0.0.46 tokenizers-0.10.3 transformers-4.12.5\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 5.3 MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.96\n",
            "Collecting datasets\n",
            "  Downloading datasets-1.16.1-py3-none-any.whl (298 kB)\n",
            "\u001b[K     |████████████████████████████████| 298 kB 5.6 MB/s \n",
            "\u001b[?25hCollecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 42.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.1.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.8.2)\n",
            "Collecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2021.11.1-py3-none-any.whl (132 kB)\n",
            "\u001b[K     |████████████████████████████████| 132 kB 45.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 40.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.6)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n",
            "\u001b[K     |████████████████████████████████| 192 kB 43.7 MB/s \n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 53.1 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |████████████████████████████████| 160 kB 60.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.6.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, fsspec, aiohttp, xxhash, datasets\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.1 asynctest-0.13.0 datasets-1.16.1 frozenlist-1.2.0 fsspec-2021.11.1 multidict-5.2.0 xxhash-2.0.2 yarl-1.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TkjsPkIosWWM"
      },
      "source": [
        "import pathlib\n",
        "import random\n",
        "import string\n",
        "import re\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import regularizers\n",
        "from datasets import load_dataset"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hd5zB1G7Y9-7",
        "outputId": "387ea602-30aa-4487-ace9-5a902e03ee20"
      },
      "source": [
        "# Detect hardware\n",
        "try:\n",
        "  tpu_resolver = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n",
        "except ValueError:\n",
        "  tpu_resolver = None\n",
        "  gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n",
        "\n",
        "# Select appropriate distribution strategy\n",
        "if tpu_resolver:\n",
        "  tf.config.experimental_connect_to_cluster(tpu_resolver)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu_resolver)\n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu_resolver)\n",
        "  print('Running on TPU ', tpu_resolver.cluster_spec().as_dict()['worker'])\n",
        "elif len(gpus) > 1:\n",
        "  strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])\n",
        "  print('Running on multiple GPUs ', [gpu.name for gpu in gpus])\n",
        "elif len(gpus) == 1:\n",
        "  strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n",
        "  print('Running on single GPU ', gpus[0].name)\n",
        "else:\n",
        "  strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n",
        "  print('Running on CPU')\n",
        "  \n",
        "print(\"Number of accelerators: \", strategy.num_replicas_in_sync)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.80.4.122:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.80.4.122:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on TPU  ['10.80.4.122:8470']\n",
            "Number of accelerators:  8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZDb8Y-6qINN0",
        "outputId": "278b7fce-bf58-4a9d-93b4-0a9625188662"
      },
      "source": [
        "!git clone 'https://github.com/nikodallanoce/HLT/'"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'HLT'...\n",
            "remote: Enumerating objects: 467, done.\u001b[K\n",
            "remote: Counting objects: 100% (467/467), done.\u001b[K\n",
            "remote: Compressing objects: 100% (380/380), done.\u001b[K\n",
            "remote: Total 467 (delta 249), reused 190 (delta 76), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (467/467), 32.94 MiB | 14.89 MiB/s, done.\n",
            "Resolving deltas: 100% (249/249), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BFIZ86xg_m-N"
      },
      "source": [
        "from transformers import BertTokenizer, BertTokenizerFast, TFMT5EncoderModel, T5TokenizerFast, XLMTokenizer, TFT5EncoderModel, DistilBertTokenizerFast, TFDistilBertModel\n",
        "\n",
        "import zipfile\n",
        "with zipfile.ZipFile(\"/content/HLT/dataset/ita-eng.zip\", 'r') as zip_ref:\n",
        "    zip_ref.extractall(\"\")"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gfgzwl95b_Zs"
      },
      "source": [
        "from tensorflow.python.ops.array_ops import boolean_mask\n",
        "# test\n",
        "# from tokenizer import *\n",
        "#import tensorflow as tf\n",
        "\n",
        "\n",
        "def create_patterns(name: str):\n",
        "    with open(name, encoding='UTF-8') as datafile:\n",
        "        sentences = datafile.readlines()\n",
        "    return sentences\n",
        "\n",
        "def extract_europarl():\n",
        "    import tarfile\n",
        "    fname = \"/content/drive/Shareddrives/HLT/datasets/it-en.tar\"\n",
        "    if fname.endswith(\"tar.gz\"):\n",
        "        tar = tarfile.open(fname, \"r:gz\")\n",
        "        tar.extractall()\n",
        "        tar.close()\n",
        "    elif fname.endswith(\"tar\"):\n",
        "        tar = tarfile.open(fname, \"r:\")\n",
        "        tar.extractall()\n",
        "        tar.close()\n",
        "\n",
        "\n",
        "def create_dataset_euparl(name: str, start : float, end : float, src: str = \"en\", dst: str = \"it\") -> (list, list):\n",
        "    extract_europarl()\n",
        "    with open(name+\".{0}\".format(src), encoding=\"UTF-8\") as datafile:\n",
        "        src_set = datafile.readlines()\n",
        "\n",
        "    with open(name+\".{0}\".format(dst), encoding=\"UTF-8\") as datafile:\n",
        "        dst_set = datafile.readlines()\n",
        "\n",
        "    \n",
        "    datasets_to_shuffle = list((zip(src_set, dst_set)))\n",
        "    #np.random.shuffle(datasets_to_shuffle)\n",
        "    start, end = int(len(src_set)*start) , int(len(src_set)*end)\n",
        "    src_set, dst_set = zip(*datasets_to_shuffle)\n",
        "    src_set = list(src_set[start:end])\n",
        "    dst_set = list(dst_set[start:end])\n",
        "\n",
        "    return src_set, dst_set\n",
        "\n",
        "\n",
        "def split_set(dataset: tf.data.Dataset,\n",
        "              tr: float = 0.8,\n",
        "              val: float = 0.1,\n",
        "              ts: float = 0.1,\n",
        "              shuffle: bool = True) -> (tf.data.Dataset, tf.data.Dataset, tf.data.Dataset):\n",
        "    if tr+val+ts != 1:\n",
        "        raise ValueError(\"Train, validation and test partition not allowed with such splits\")\n",
        "\n",
        "    dataset_size = dataset.cardinality().numpy()\n",
        "    if shuffle:\n",
        "        dataset = dataset.shuffle(dataset_size)\n",
        "\n",
        "    tr_size = int(tr * dataset_size)\n",
        "    val_size = int(val * dataset_size)\n",
        "\n",
        "    tr_set = dataset.take(tr_size)\n",
        "    val_set = dataset.skip(tr_size).take(val_size)\n",
        "    ts_set = dataset.skip(tr_size).skip(val_size)\n",
        "    return tr_set, val_set, ts_set\n",
        "\n",
        "\n",
        "def make_batches(dataset_src_dst: tf.data.Dataset, batch_size: int):\n",
        "    return dataset_src_dst.cache().batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "\n",
        "def create_dataset_anki(name: str, preprocessed:bool):\n",
        "    with open(name, encoding=\"UTF-8\") as datafile:\n",
        "        src_set = list()\n",
        "        dst_set = list()\n",
        "        for sentence in datafile:\n",
        "            sentence = sentence.split(\"\\t\")\n",
        "            src_set.append(sentence[0])\n",
        "            if preprocessed:\n",
        "                dst_set.append(sentence[1].split(\"\\n\")[0])\n",
        "            else:\n",
        "                dst_set.append(sentence[1])\n",
        "\n",
        "    return src_set, dst_set\n",
        "\n",
        "def dataset_merged(ds1, ds2):\n",
        "    src_ds1, dst_ds1 = ds1\n",
        "    src_ds2, dst_ds2 = ds2\n",
        "    src = src_ds1 + src_ds2\n",
        "    dst = dst_ds1 + dst_ds2\n",
        "\n",
        "    return src, dst    "
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcRp7VcQ5m6g"
      },
      "source": [
        "BUFFER_SIZE = 20000\n",
        "DS_SIZE= 2**17\n",
        "sl = 90\n",
        "def format_dataset(src, trg):\n",
        "    return ({\"encoder_inputs\": src, \"decoder_inputs\": trg[:, :-1]}, trg[:, 1:])\n",
        "\n",
        "def make_dataset(dataset, batch_size):\n",
        "    dataset = dataset.batch(batch_size)\n",
        "    dataset = dataset.map(format_dataset)\n",
        "    return dataset.prefetch(tf.data.experimental.AUTOTUNE).cache()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C6Mh4HYUTZXX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241,
          "referenced_widgets": [
            "940f7939964a42c6ad2a82d6ae3b091f",
            "7aecd392fe514e22b024b0f6ea4e8eae",
            "d3ee2e58ecd546779572760996857c1d",
            "357d66b3c39845e7b91aea7e9df6c7c6",
            "1751801ea9d044fea02f9da9c28db1e3",
            "a062984d177645bc98069b2e025f4d39",
            "9a95637be3f1464281a1aefa70a34f8e",
            "ce9fb2f1856a4a38b964b45fe749997b",
            "9c82c009a6824a44bec384fc0cb1797c",
            "b9c8fe2aa88642cfa3b93bb05449a24d",
            "b84031172f2544e08fc3b937378f55da",
            "6c3b1a3aecfe47b5aa93f34dbd757e8b",
            "e83a91d035f64dc085de5f0b1ffbc48d",
            "e29db46551d54469bfbad591d8492e7d",
            "eb0d9a7715fd4b90bc08c54c72c344b6",
            "e73a88c265b7433288e30c8909818828",
            "c1cfc00cf0e847629dfc94e1b3ab70c7",
            "fb7776b48bb54ec88ac4e76a2a0f0ac8",
            "887cc78501424f5bbd0146d5824595a5",
            "f2eac8eab2db48e49b191ef9fdc64485",
            "fdfe94cd6aef4f4e8857b25b1422e4ee",
            "d64253b27bfe446dba1873ef68017666",
            "8985decb7b9042b7ab17fd438c77a7a4",
            "31ff1e662b1e426a9f475299e0606165",
            "899c536e90394cc291e9b71024ff67da",
            "4f3288ad853e42e59c04ad9590b78bb4",
            "63f3063734c040a1a151676eedbee6a5",
            "d769cc15ce3e4364b04b125a59f201b8",
            "99e973ad66b547538987881c28591324",
            "b85016da309542cc8c39b7c588b5306d",
            "f0410fe0c95844738760e5da327a9b63",
            "5d9393c00a794be4aa74e676117823ae",
            "07d513487e1c4fb39d661eb8bf1415e9",
            "25637fbda4ce4c65bf96b21f461149b9",
            "5b3eac0bc02e497296c72668857779fe",
            "6a33798ccf2544ba89c9147db20e84e9",
            "6b6b965a6a1d4c84abae85cd60b419da",
            "bb279bc90fca4b198df83f182a46ba4c",
            "bf35fbde55c9468c8bec2bf0bb1859da",
            "27012e6998944073abb2594cda29381c",
            "43986ec3ea8848feb3b49298aaf77699",
            "7e940f47c371456da12e996bd2efcbe5",
            "9cd345b312c9491fa0de0e6cabc7d00a",
            "4d79a6cf81f141079724611c0afd83f8",
            "60514887103243f484c8215e7d8ed523",
            "438c5f356c9a4720a8e6f9dec4b76ef7",
            "56b84936f9a240ce96389b29696944d5",
            "9a4ca384567e43dd9a7af2f99c53d19d",
            "417241363a534a98a831d403ad002104",
            "4fc27818b4534bdcb5c21522988b09a4",
            "3707e92dc8f34c179c33d2aa665d4cf6",
            "c4d51f645eb8402eb5ad99c8879c415a",
            "985d3dc05f194dd99e329a338980242d",
            "35d48c41b0984e65875399a6eb1910a0",
            "1818ae8b21964fdd9a9465bceca10acc",
            "6a791b2f59bb42f6a04e159dbadf2292",
            "f4e7a05887f74aef8ea992bbf79a21d9",
            "f76ee5d9a70c4fda870ca4d3fcc1b6d5",
            "7fd62e8aa6524f1da5bd234ea61a88e5",
            "345ec00a503240a5b0bdcc6923ee71a9",
            "36b2de4e605d401ea0605383db4c2f0e",
            "db8d5d08ae33469f884343c8e9d0fde2",
            "89981cc6d6c84dc9bf6cc6e9365bd427",
            "80daa77ab1794ba8b36ee2c8c1bfb988",
            "aa2e7a81ff5041ccb94bf7e4ab8aa28d",
            "1ae0cf5818d64b97a1d04ae900a21436",
            "88c530eff74c481ebfa6a963b5148e8c",
            "b43a700a0af4467e8642801c88a15b3f",
            "4693ba52d57f4206b9373ab8aabf4f9e",
            "2a7751448cb34e7d87fadab5ca419324",
            "dcb837cd796f4cb398061709663f74fa",
            "98d424e344574b2d9bea88db23e543fe",
            "c365a69a49d546dd92c905f38e963dfd",
            "1aaa3503ada94f05af04e430920c8c13",
            "54294d943c4a4a1aad5aab5e743a99b7",
            "aab38013a26346bca32d2cd34a63690c",
            "0cd1675a52a34ccb920d24d3b747c51a"
          ]
        },
        "outputId": "6ce459a0-d972-4383-a915-431d65cf9037"
      },
      "source": [
        "from transformers import XLNetTokenizerFast, ByT5Tokenizer, GPT2TokenizerFast, TFMT5EncoderModel, RobertaTokenizerFast\n",
        "source_src= \"google/t5-v1_1-small\" #\"DeepESP/gpt2-spanish\"\n",
        "target_src = \"dbmdz/bert-base-italian-cased\"\n",
        "tokenizer_source = T5TokenizerFast.from_pretrained(source_src)\n",
        "tokenizer_target = BertTokenizerFast.from_pretrained(target_src)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "940f7939964a42c6ad2a82d6ae3b091f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/773k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6c3b1a3aecfe47b5aa93f34dbd757e8b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.74k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8985decb7b9042b7ab17fd438c77a7a4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.81k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "25637fbda4ce4c65bf96b21f461149b9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/537 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "60514887103243f484c8215e7d8ed523",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/230k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6a791b2f59bb42f6a04e159dbadf2292",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/59.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "88c530eff74c481ebfa6a963b5148e8c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/433 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hqZrNDoTcJqr"
      },
      "source": [
        "# test\n",
        "#source_set, target_set = dataset_merged(create_dataset_anki(\"ita.txt\", False), create_dataset_euparl(\"europarl-v7.it-en\", size=0.22))\n",
        "#source_set, target_set = create_dataset_euparl(\"europarl-v7.it-en\", size=0.22)\n",
        "\n",
        "sequence_length = sl\n",
        "# Create the tokenizers and get the number of tokens\n",
        "#logging.set_verbosity_error()  # suppress warnings for transformers\n",
        "#source_set, target_set = create_dataset_anki(\"ita.txt\", False)\n",
        "tokenizer_source = T5TokenizerFast.from_pretrained(source_src)\n",
        "tokenizer_target = BertTokenizerFast.from_pretrained(target_src)\n",
        "  \n",
        "v_size_src = tokenizer_source.vocab_size\n",
        "\n",
        "v_size_trg = tokenizer_target.vocab_size\n",
        "\n",
        "# Tokenize the dataset\n",
        "def tokenize(entire_set):\n",
        "\n",
        "  source_set, target_set = entire_set\n",
        "  tokens_source = tokenizer_source(source_set, truncation=True, padding=\"max_length\",\n",
        "                              return_tensors=\"tf\", max_length=sequence_length).data[\"input_ids\"]\n",
        "  tokens_source = tf.cast(tokens_source, dtype=tf.int32)                            \n",
        "  tokens_target = tokenizer_target(target_set, add_special_tokens=True,\n",
        "                              truncation=True, padding=\"max_length\",\n",
        "                              return_tensors=\"tf\", max_length=sequence_length + 1).data[\"input_ids\"]\n",
        "  tokens_target = tf.cast(tokens_target, dtype=tf.int32)\n",
        "  return tokens_source, tokens_target                   "
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NX0tfKjVbEhe"
      },
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices(tokenize(create_dataset_anki(\"ita.txt\", False)))\n",
        "dataset = dataset.concatenate(tf.data.Dataset.from_tensor_slices(tokenize(create_dataset_euparl(\"europarl-v7.it-en\", start=0, end=0.3))))\n",
        "dataset = dataset.concatenate(tf.data.Dataset.from_tensor_slices(tokenize(create_dataset_euparl(\"europarl-v7.it-en\", start=0.3, end=0.7))))\n",
        "dataset = dataset.concatenate(tf.data.Dataset.from_tensor_slices(tokenize(create_dataset_euparl(\"europarl-v7.it-en\", start=0.7, end=1))))"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ad2vywWoMh5d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47fc94ea-d5aa-421f-ee58-9f70dedcdecb"
      },
      "source": [
        "#dataset = tf.data.Dataset.from_tensor_slices((tokens_source, tokens_target))\n",
        "#dataset = dataset.concatenate(dataset)  # build the tf dataset\n",
        "tr_set, val_set, ts_set = split_set(dataset, 0.9, 0.05, 0.05)  # split the tf dataset\n",
        "print(len(dataset))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2261155\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hnlz5YW3gkIo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0db88409-5bd1-45e4-b088-a0935e0ad1fe"
      },
      "source": [
        "tokenizer_source.vocab_size"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32100"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EyPxlll0xhnX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2fd5b97f-065c-48b4-dec4-fe5452edbd54"
      },
      "source": [
        "batch_size =  16 * strategy.num_replicas_in_sync\n",
        "with strategy.scope():\n",
        "  train_ds = make_dataset(tr_set, batch_size)\n",
        "  val_ds = make_dataset(val_set, batch_size)\n",
        "train_ds  "
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<CacheDataset shapes: ({encoder_inputs: (None, 90), decoder_inputs: (None, 90)}, (None, 90)), types: ({encoder_inputs: tf.int32, decoder_inputs: tf.int32}, tf.int32)>"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U6DnsSvjsWWR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a5d7437a-7811-4552-dcca-9f4dc0b78b21"
      },
      "source": [
        "#pe = PositionalEmbedding(30, 100, 128)\n",
        "\n",
        "for inputs, targets in train_ds:\n",
        "    print(inputs)\n",
        "    print(targets)\n",
        "    break;"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'encoder_inputs': <tf.Tensor: shape=(128, 90), dtype=int32, numpy=\n",
            "array([[1615,  410,   34, ...,    0,    0,    0],\n",
            "       [ 101,  398, 2459, ...,    0,    0,    0],\n",
            "       [  86,  455,   12, ...,    0,    0,    0],\n",
            "       ...,\n",
            "       [  86,   48, 1445, ...,    0,    0,    0],\n",
            "       [ 148,   31,  195, ...,    0,    0,    0],\n",
            "       [ 101,   33, 2508, ...,    0,    0,    0]], dtype=int32)>, 'decoder_inputs': <tf.Tensor: shape=(128, 90), dtype=int32, numpy=\n",
            "array([[  102,  1529,  1307, ...,     0,     0,     0],\n",
            "       [  102,  2689,  3691, ...,     0,     0,     0],\n",
            "       [  102,   435,  1027, ...,     0,     0,     0],\n",
            "       ...,\n",
            "       [  102,   369,  4130, ...,     0,     0,     0],\n",
            "       [  102,   313,  7894, ...,     0,     0,     0],\n",
            "       [  102,  4231, 24603, ...,     0,     0,     0]], dtype=int32)>}\n",
            "tf.Tensor(\n",
            "[[ 1529  1307   527 ...     0     0     0]\n",
            " [ 2689  3691   406 ...     0     0     0]\n",
            " [  435  1027   120 ...     0     0     0]\n",
            " ...\n",
            " [  369  4130  1307 ...     0     0     0]\n",
            " [  313  7894   993 ...     0     0     0]\n",
            " [ 4231 24603   980 ...     0     0     0]], shape=(128, 90), dtype=int32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3iZAS6kYsWWR"
      },
      "source": [
        "## Building the model\n",
        "\n",
        "Our sequence-to-sequence Transformer consists of a `TransformerEncoder`\n",
        "and a `TransformerDecoder` chained together. To make the model aware of word order,\n",
        "we also use a `PositionalEmbedding` layer.\n",
        "\n",
        "The source sequence will be pass to the `TransformerEncoder`,\n",
        "which will produce a new representation of it.\n",
        "This new representation will then be passed\n",
        "to the `TransformerDecoder`, together with the target sequence so far (target words 0 to N).\n",
        "The `TransformerDecoder` will then seek to predict the next words in the target sequence (N+1 and beyond).\n",
        "\n",
        "A key detail that makes this possible is causal masking\n",
        "(see method `get_causal_attention_mask()` on the `TransformerDecoder`).\n",
        "The `TransformerDecoder` sees the entire sequences at once, and thus we must make\n",
        "sure that it only uses information from target tokens 0 to N when predicting token N+1\n",
        "(otherwise, it could use information from the future, which would\n",
        "result in a model that cannot be used at inference time)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQUNEN22Oh90"
      },
      "source": [
        "dropout_rate= 0.2"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SUtqG1Wik3L",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 671,
          "referenced_widgets": [
            "327e0394f2e4479fa6c9019f9240382f",
            "1bf3adfb4ff245708e8dba72795b5c96",
            "57f332fd467e4c1ca2d21afbe7d4b643",
            "87d9eec07ddc47bb91910b6838af90db",
            "0207d92f40884ffab8ae4697a7109b78",
            "07530cbe5ce448ca92d24a66d1a6d7d9",
            "6a564baaeb4043bf882a7259df2dfe87",
            "33dc08585df546479426ac70de917e46",
            "cbf6f34d27db43bb86d372e123e5b5ed",
            "4bf9db5d046b4a6ba3b01cc11b8d58a2",
            "39d0b6ce158048849dbd7de3ef9d6232"
          ]
        },
        "outputId": "d3262e4e-e944-4559-c7b0-7be11432c617"
      },
      "source": [
        "from transformers import TFXLNetModel, TFGPT2Model, TFBertModel, TFRobertaModel\n",
        "with strategy.scope():\n",
        "  encoder = TFT5EncoderModel.from_pretrained(source_src)\n",
        "print(encoder.config)  "
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "327e0394f2e4479fa6c9019f9240382f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/294M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at google/t5-v1_1-small were not used when initializing TFT5EncoderModel: ['decoder', 'lm_head']\n",
            "- This IS expected if you are initializing TFT5EncoderModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFT5EncoderModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFT5EncoderModel were initialized from the model checkpoint at google/t5-v1_1-small.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5EncoderModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "T5Config {\n",
            "  \"_name_or_path\": \"google/t5-v1_1-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 1024,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"gated-gelu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"num_decoder_layers\": 8,\n",
            "  \"num_heads\": 6,\n",
            "  \"num_layers\": 8,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"tie_word_embeddings\": false,\n",
            "  \"transformers_version\": \"4.12.5\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aRKVNnkzsWWR"
      },
      "source": [
        "class TransformerEncoder(layers.Layer):\n",
        "    def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n",
        "        super(TransformerEncoder, self).__init__(**kwargs)\n",
        "        self.embed_dim = embed_dim\n",
        "        self.dense_dim = dense_dim\n",
        "        self.num_heads = num_heads\n",
        "\n",
        "        self.attention = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim\n",
        "        )\n",
        "        self.dense_proj = keras.Sequential(\n",
        "            [layers.Dense(dense_dim, activation=\"elu\"), layers.Dense(embed_dim),]\n",
        "        )\n",
        "        self.layernorm_1 = layers.LayerNormalization()\n",
        "        self.layernorm_2 = layers.LayerNormalization()\n",
        "        self.supports_masking = True\n",
        "\n",
        "    def call(self, inputs, mask=None):\n",
        "        if mask is not None:  \n",
        "            padding_mask = tf.cast(mask[:, tf.newaxis, tf.newaxis, :], dtype=\"int32\")\n",
        "        else:\n",
        "            assert False\n",
        "        attention_output = self.attention(\n",
        "            query=inputs, value=inputs, key=inputs, attention_mask=padding_mask\n",
        "        )\n",
        "        proj_input = self.layernorm_1(inputs + attention_output)\n",
        "        proj_output = self.dense_proj(proj_input)\n",
        "        return self.layernorm_2(proj_input + proj_output)\n",
        "\n",
        "\n",
        "class PositionalEmbedding(layers.Layer):\n",
        "  def __init__(self, sequence_length, vocab_size, embed_dim, **kwargs):\n",
        "      super(PositionalEmbedding, self).__init__(**kwargs)\n",
        "      self.token_embeddings = layers.Embedding(\n",
        "          input_dim= vocab_size, output_dim=embed_dim\n",
        "      )\n",
        "      self.position_embeddings = layers.Embedding(\n",
        "          input_dim=sequence_length, output_dim=embed_dim\n",
        "      )\n",
        "      self.sequence_length = sequence_length\n",
        "      self.vocab_size = vocab_size\n",
        "      self.embed_dim = embed_dim\n",
        "\n",
        "  def call(self, inputs):\n",
        "      length = tf.shape(inputs)[-1] \n",
        "      positions = tf.range(start=0, limit=length, delta=1)\n",
        "      embedded_tokens = self.token_embeddings(inputs)\n",
        "      embedded_positions = self.position_embeddings(positions)\n",
        "      return embedded_tokens + embedded_positions\n",
        "\n",
        "  def compute_mask(self, inputs, mask=None):\n",
        "      return tf.math.not_equal(inputs, 0)\n",
        "\n",
        "\n",
        "class TransformerDecoder(layers.Layer):\n",
        "    def __init__(self, embed_dim, latent_dim, num_heads, **kwargs):\n",
        "        super(TransformerDecoder, self).__init__(**kwargs)\n",
        "        self.embed_dim = embed_dim\n",
        "        self.latent_dim = latent_dim\n",
        "        self.num_heads = num_heads\n",
        "        self.attention_1 = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim, dropout=dropout_rate\n",
        "        )\n",
        "        self.attention_2 = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim, dropout=dropout_rate\n",
        "        )\n",
        "        self.dense_proj = keras.Sequential(\n",
        "            [layers.Dense(latent_dim, activation=\"elu\"), layers.Dropout(dropout_rate), layers.Dense(embed_dim)])\n",
        "        #self.dense_proj_f = keras.Sequential(\n",
        "        #    [layers.Dense(latent_dim, activation=\"elu\"), layers.Dropout(dropout_rate), layers.Dense(embed_dim)])\n",
        "        self.layernorm_1 = layers.LayerNormalization()\n",
        "        #self.layernorm_1_f = layers.LayerNormalization()\n",
        "        self.layernorm_2 = layers.LayerNormalization()\n",
        "        self.layernorm_3 = layers.LayerNormalization()\n",
        "\n",
        "        #self.dropout_1 = layers.Dropout(dropout_rate)\n",
        "        #self.dropout_2 = layers.Dropout(dropout_rate)\n",
        "\n",
        "        self.resid1=layers.Add()\n",
        "        self.resid2=layers.Add()\n",
        "        self.resid3=layers.Add()\n",
        "        self.supports_masking = True\n",
        "\n",
        "    def call(self, inputs, encoder_outputs, mask=None):\n",
        "        causal_mask = self.get_causal_attention_mask(inputs)\n",
        "        if mask is not None:\n",
        "            padding_mask = tf.cast(mask[:, tf.newaxis, :], dtype=\"int32\")\n",
        "            padding_mask = tf.minimum(padding_mask, causal_mask)\n",
        "\n",
        "        attention_output_1 = self.attention_1(\n",
        "            query=inputs, value=inputs, key=inputs, attention_mask=causal_mask\n",
        "        )\n",
        "        \n",
        "        #attention_output_1 = self.dropout_1(attention_output_1)\n",
        "        out_1 = self.layernorm_1(self.resid1([inputs, attention_output_1]))\n",
        "        #proj_output_f = self.dense_proj_f(out_1)\n",
        "        #out_1 = self.layernorm_1_f(layers.Add()([out_1, proj_output_f]))\n",
        "\n",
        "        attention_output_2 = self.attention_2(\n",
        "            query=out_1,\n",
        "            value=encoder_outputs,\n",
        "            key=encoder_outputs,\n",
        "            attention_mask=padding_mask\n",
        "        )\n",
        "        #attention_output_2 = self.dropout_2(attention_output_2)\n",
        "        out_2 = self.layernorm_2(self.resid2([out_1, attention_output_2]))\n",
        "\n",
        "        proj_output = self.dense_proj(out_2)\n",
        "        #proj_output = self.dropout_1(proj_output)\n",
        "        return self.layernorm_3(self.resid3([out_2, proj_output]))\n",
        "\n",
        "    def get_causal_attention_mask(self, inputs):\n",
        "        input_shape = tf.shape(inputs)\n",
        "        batch_size, sequence_length = input_shape[0], input_shape[1]\n",
        "        i = tf.range(sequence_length)[:, tf.newaxis]\n",
        "        j = tf.range(sequence_length)\n",
        "        mask = tf.cast(i >= j, dtype=\"int32\")\n",
        "        mask = tf.reshape(mask, (1, input_shape[1], input_shape[1]))\n",
        "        mult = tf.concat(\n",
        "            [tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)],\n",
        "            axis=0,\n",
        "        )\n",
        "        return tf.tile(mask, mult)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tWAPsdSfsWWS"
      },
      "source": [
        "Next, we assemble the end-to-end model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CzLJYOshsWWS"
      },
      "source": [
        "latent_dim = 2048\n",
        "num_heads = 8\n",
        "\n",
        "def create_model(encoder, embed_dim, v_size_trg):\n",
        "  encoder_inputs = tf.keras.Input(shape=(None,), dtype=\"int32\", name=\"encoder_inputs\")\n",
        "  outputs = encoder(encoder_inputs)\n",
        "  encoder_outputs = outputs.last_hidden_state\n",
        "\n",
        "  decoder_inputs = tf.keras.Input(shape=(None,), dtype=\"int32\", name=\"decoder_inputs\")\n",
        "  encoded_seq_inputs = tf.keras.Input(shape=(None, embed_dim), name=\"decoder_state_inputs\")\n",
        "  x = PositionalEmbedding(sequence_length, v_size_trg, embed_dim)(decoder_inputs)\n",
        "  x = layers.Dropout(dropout_rate)(x)\n",
        "  for i in range(8):\n",
        "    x = TransformerDecoder(embed_dim, latent_dim, num_heads)(x, encoded_seq_inputs)\n",
        "\n",
        "  decoder_outputs = layers.Dense(v_size_trg, activation=\"softmax\")(x)\n",
        "  decoder = tf.keras.Model([decoder_inputs, encoded_seq_inputs], decoder_outputs)\n",
        "\n",
        "  decoder_outputs = decoder([decoder_inputs, encoder_outputs])\n",
        "  transformer = tf.keras.Model(\n",
        "      [encoder_inputs, decoder_inputs], decoder_outputs, name=\"transformer\"\n",
        "  )\n",
        "  return transformer"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mad4pECqsWWS"
      },
      "source": [
        "## Training our model\n",
        "\n",
        "We'll use accuracy as a quick way to monitor training progress on the validation data.\n",
        "Note that machine translation typically uses BLEU scores as well as other metrics, rather than accuracy.\n",
        "\n",
        "Here we only train for 1 epoch, but to get the model to actually converge\n",
        "you should train for at least 30 epochs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGqZaPCi3mqU"
      },
      "source": [
        "with strategy.scope():\n",
        "\n",
        "  model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath='/content/drive/Shareddrives/HLT/t5v11small_giga_tot.h5',\n",
        "    monitor=\"val_loss\",\n",
        "    verbose=0,\n",
        "    save_weights_only=True,\n",
        "    mode=\"auto\",\n",
        "    save_freq=\"epoch\",\n",
        "    overwrite=True)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ivBYMWvNAFE"
      },
      "source": [
        "class CustomSchedule(tf.keras.callbacks.LearningRateScheduler):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "\n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "\n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApHH-su2-fW9"
      },
      "source": [
        "d_model = 512\n",
        "with strategy.scope():\n",
        "  transformer = create_model(encoder, d_model, v_size_trg)\n",
        "  learning_rate = CustomSchedule(d_model)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VR8fzpHEsWWS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 779
        },
        "outputId": "b832b50e-6c56-402c-a8b9-57de008e78fd"
      },
      "source": [
        "epochs = 8  # This should be at least 30 for convergence\n",
        "\n",
        "with strategy.scope():\n",
        "  opt = tf.keras.optimizers.Adam()\n",
        "  train_ds = train_ds.shuffle(10**6)\n",
        "  transformer.summary()\n",
        "  transformer.compile(opt, loss = \"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n",
        "transformer.fit(train_ds, epochs=1, validation_data = val_ds, callbacks=[learning_rate_scheduler , model_checkpoint_callback], shuffle=True)\n",
        "#♦transformer.save_weights('/content/drive/Shareddrives/HLT/t5v11small_giga_tot.h5', overwrite=True)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"transformer\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " encoder_inputs (InputLayer)    [(None, None)]       0           []                               \n",
            "                                                                                                  \n",
            " decoder_inputs (InputLayer)    [(None, None)]       0           []                               \n",
            "                                                                                                  \n",
            " tft5_encoder_model (TFT5Encode  TFBaseModelOutput(l  35332800   ['encoder_inputs[0][0]']         \n",
            " rModel)                        ast_hidden_state=(N                                               \n",
            "                                one, None, 512),                                                  \n",
            "                                 hidden_states=None                                               \n",
            "                                , attentions=None)                                                \n",
            "                                                                                                  \n",
            " model_3 (Functional)           (None, None, 31102)  183170430   ['decoder_inputs[0][0]',         \n",
            "                                                                  'tft5_encoder_model[4][0]']     \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 218,503,230\n",
            "Trainable params: 218,503,230\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-120e2e67c197>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"sparse_categorical_crossentropy\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"accuracy\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m   \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval_ds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m   \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/Shareddrives/HLT/t5v11small_giga_tot.h5'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiltered_tb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m       \u001b[0;32mdel\u001b[0m \u001b[0mfiltered_tb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mset_model\u001b[0;34m(self, model)\u001b[0m\n\u001b[1;32m    283\u001b[0m       \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_history\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 285\u001b[0;31m       \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'CustomSchedule' object has no attribute 'set_model'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fEdpAMpbVLou"
      },
      "source": [
        "tf.keras.utils.plot_model(\n",
        "    transformer, to_file='model.png', show_shapes=True, dpi=90\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vj4TsHwysWWS"
      },
      "source": [
        "## Decoding test sentences\n",
        "\n",
        "Finally, let's demonstrate how to translate brand new English sentences.\n",
        "We simply feed into the model the vectorized English sentence\n",
        "as well as the target token `\"[start]\"`, then we repeatedly generated the next token, until\n",
        "we hit the token `\"[end]\"`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TEInfs0peDfc"
      },
      "source": [
        "import shutil\n",
        "shutil.move(\"/content/translator.h5\", \"/content/drive/Shareddrives/HLT/t5v11small_giga.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bw_Et-rpJKuC"
      },
      "source": [
        "transformer.save_weights('/content/drive/Shareddrives/HLT/t5v11small_giga.h5', overwrite=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZtPZMFHU3Sn"
      },
      "source": [
        "#tokenizer_it = BertTokenizer.from_pretrained(ita_src)\n",
        "with strategy.scope():\n",
        "  transformer = create_model(encoder, 512, v_size_trg)\n",
        "  transformer.load_weights('/content/drive/Shareddrives/HLT/t5v11small_giga_tot.h5')"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiB9dW8VsWWS"
      },
      "source": [
        "def decode_sequence(input_sentence, tokenizer_source, tokenizer_target, transformer):\n",
        "    #tokenized_input_sentence=input_sentence\n",
        "    tokenized_input_sentence = tokenizer_source(input_sentence, return_tensors='tf', add_special_tokens=True, max_length = sequence_length, padding='max_length', truncation=True).data[\"input_ids\"]\n",
        "    decoded_sentence = \"[CLS]\"\n",
        "    list_tokens=[decoded_sentence]\n",
        "    for i in range(sequence_length):\n",
        "\n",
        "        decoded_sentence = tokenizer_target.convert_tokens_to_string(list_tokens)\n",
        "        tokenized_target_sentence = tokenizer_target(decoded_sentence, return_tensors='tf', add_special_tokens=False, max_length = sequence_length, padding='max_length').data['input_ids']\n",
        "        predictions = transformer([tokenized_input_sentence, tokenized_target_sentence])\n",
        "        sampled_token_index = np.argmax(predictions[0, i, :])\n",
        "        sampled_token = tokenizer_target.ids_to_tokens[sampled_token_index] #spa_index_lookup[sampled_token_index]\n",
        "        \n",
        "        #decoded_sentence += sampled_token\n",
        "       \n",
        "        if sampled_token == \"[SEP]\":\n",
        "          decoded_sentence = tokenizer_target.convert_tokens_to_string(list_tokens[1:])\n",
        "          break\n",
        "        list_tokens.append(sampled_token)\n",
        "    \n",
        "    return list_tokens, decoded_sentence\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N6yhcfI8rUHV"
      },
      "source": [
        "from transformers import GPT2Tokenizer, TFGPT2Model, T5Tokenizer, BertTokenizer, XLNetTokenizer\n",
        "\n",
        "tokenizer_source1 = T5TokenizerFast.from_pretrained(source_src)\n",
        "tokenizer_target1 = BertTokenizer.from_pretrained(target_src)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hhBNEDomEfu-"
      },
      "source": [
        "to_translate = \"I often go to the sea on the weekend, but this week I'm busy and I can't go there.\"\n",
        "with strategy.scope():\n",
        "    tokens, translated = decode_sequence(to_translate, tokenizer_source1, tokenizer_target1, transformer)\n",
        "print([to_translate, translated])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FctHyhS69jzY"
      },
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "references = []\n",
        "start, end= 349950, 350000\n",
        "\n",
        "for sent, frase in zip(en_set[start:end], it_set[start:end]):\n",
        "    \n",
        "    frase = tokenizer_it(frase, add_special_tokens=False).data['input_ids']\n",
        "    frase = tokenizer_it.convert_ids_to_tokens(frase)\n",
        "    frase = tokenizer_it.convert_tokens_to_string(frase)\n",
        "    references.append((sent, frase.split()))\n",
        "\n",
        "score = 0\n",
        "with strategy.scope():\n",
        "  for en, it in tqdm_notebook(references):\n",
        "      _, translated = decode_sequence(en)\n",
        "      #print([translated, it])\n",
        "      score += sentence_bleu([it], translated.split())\n",
        "      print((it, translated.split()))\n",
        "score = score / len(references)\n",
        "print(score)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RGnB1edtBUQV"
      },
      "source": [
        "score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YcUhRmPgAEBZ"
      },
      "source": [
        "epochs = 5  # This should be at least 30 for convergence\n",
        "\n",
        "with strategy.scope():\n",
        "  opt = tf.keras.optimizers.Adam(learning_rate = 0.00005)\n",
        "  #opt = tf.keras.optimizers.RMSprop(learning_rate=0.0001)\n",
        "#  opt = tf.keras.optimizers.SGD(learning_rate = 0.05, momentum = 0.65)\n",
        "  transformer.compile(opt, loss = \"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n",
        "transformer.fit(train_ds, epochs=epochs, validation_data = val_ds, callbacks=[model_checkpoint_callback])"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}