PR_TXT_Gen.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "collapsed_sections": [
        "RNLxnMo360uM",
        "141URoNtKpdP",
        "nGyRH6TyKYHa",
        "ZZnMvg5qwTd_"
      ],
      "authorship_tag": "ABX9TyPTy8x2yMOdrr0HcjKOs3bn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "243d367a86494afda88e98d91c1437a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_46cff285f4444f309306ec1387b432db",
              "IPY_MODEL_c70c1acfaef74c1f95a14009c879e73e",
              "IPY_MODEL_b1b02dadea9940de9d4191d867209cd1"
            ],
            "layout": "IPY_MODEL_82667dbb8de847eea9efe089cf42b63a"
          }
        },
        "46cff285f4444f309306ec1387b432db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f10e96bf0434515bd1de2e81b10e01c",
            "placeholder": "​",
            "style": "IPY_MODEL_123da6dad0a044d8b9dd63cce0237a8b",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "c70c1acfaef74c1f95a14009c879e73e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_608e02fd8c874966b66197957a219b66",
            "max": 689,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f49cc366148143bab7e8bb031c88fd9e",
            "value": 689
          }
        },
        "b1b02dadea9940de9d4191d867209cd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8690cfe9ddf348879092d6737424579b",
            "placeholder": "​",
            "style": "IPY_MODEL_d0d1d8bb5214411fa93b455ac702716d",
            "value": " 689/689 [00:00&lt;00:00, 19.6kB/s]"
          }
        },
        "82667dbb8de847eea9efe089cf42b63a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f10e96bf0434515bd1de2e81b10e01c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "123da6dad0a044d8b9dd63cce0237a8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "608e02fd8c874966b66197957a219b66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f49cc366148143bab7e8bb031c88fd9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8690cfe9ddf348879092d6737424579b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d0d1d8bb5214411fa93b455ac702716d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c2c97690bd784468949c9a32d99add61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_384a6723c7dc4749815f71b44d02a939",
              "IPY_MODEL_121db7d836184e6993fcd7791deaf0c0",
              "IPY_MODEL_2fd172aaed68434a861a11dbd82799c7"
            ],
            "layout": "IPY_MODEL_bbc10f3304294130bf9e799aee197ce3"
          }
        },
        "384a6723c7dc4749815f71b44d02a939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc0f74d1366b4c0ab8d1769df0746b53",
            "placeholder": "​",
            "style": "IPY_MODEL_ef63bdd4cf334d31a3426fcee007a753",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "121db7d836184e6993fcd7791deaf0c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba701748a42249fd82b104866c13d054",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_03333c0e32624fe5bd57ebdca7a237d2",
            "value": 1042301
          }
        },
        "2fd172aaed68434a861a11dbd82799c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e2512d51d2c94877bc924bf65d7a9f6d",
            "placeholder": "​",
            "style": "IPY_MODEL_ea2e5692132844efb2457ff66292642e",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 9.08MB/s]"
          }
        },
        "bbc10f3304294130bf9e799aee197ce3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc0f74d1366b4c0ab8d1769df0746b53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef63bdd4cf334d31a3426fcee007a753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba701748a42249fd82b104866c13d054": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "03333c0e32624fe5bd57ebdca7a237d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e2512d51d2c94877bc924bf65d7a9f6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea2e5692132844efb2457ff66292642e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba4fa5d988754058b4bff6527d43ea2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_04797c0f4eb846c3ae27a12f3a1dea96",
              "IPY_MODEL_82649d0a81d145ba9a572ed3ba992fe2",
              "IPY_MODEL_94a950ee0aba4b7da0cfa31cece865a1"
            ],
            "layout": "IPY_MODEL_b53bf3ee2c6049a9be5b658c797397b7"
          }
        },
        "04797c0f4eb846c3ae27a12f3a1dea96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5892668ab96641eea35a65c2b58582f2",
            "placeholder": "​",
            "style": "IPY_MODEL_571c8041b9614ac5aacfc12c2f27b8ba",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "82649d0a81d145ba9a572ed3ba992fe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c42bca06989c42c7b4b4eaf9ef203e4b",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1522d99aa8654e33b362226732c9b821",
            "value": 456318
          }
        },
        "94a950ee0aba4b7da0cfa31cece865a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6075082aba6a4552b19b66bb3bbf195d",
            "placeholder": "​",
            "style": "IPY_MODEL_83db7e1281394854aa54aa27803b6ddd",
            "value": " 456k/456k [00:00&lt;00:00, 3.25MB/s]"
          }
        },
        "b53bf3ee2c6049a9be5b658c797397b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5892668ab96641eea35a65c2b58582f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "571c8041b9614ac5aacfc12c2f27b8ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c42bca06989c42c7b4b4eaf9ef203e4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1522d99aa8654e33b362226732c9b821": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6075082aba6a4552b19b66bb3bbf195d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83db7e1281394854aa54aa27803b6ddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6130a6d8a1d24175ba52014bf793f24c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_78a8342356174d67a362f1f4df0d200f",
              "IPY_MODEL_a6845b6e488f467292e71125d892475b",
              "IPY_MODEL_3f9410bfb41e401198b9c60575b2fce6"
            ],
            "layout": "IPY_MODEL_90fba3627bd24d2aa3ea3438d256aea2"
          }
        },
        "78a8342356174d67a362f1f4df0d200f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9d032b7b09b4655bf6857993973d3f8",
            "placeholder": "​",
            "style": "IPY_MODEL_a72b33ac68d3482c861ebaebe3c2ac5b",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "a6845b6e488f467292e71125d892475b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_24ebf05cf1754766be7545b910b479ce",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f800d9a988214f46957e3063ecaa3fbd",
            "value": 1355256
          }
        },
        "3f9410bfb41e401198b9c60575b2fce6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1446d2d11c8e4ba88fd811cad715dd3c",
            "placeholder": "​",
            "style": "IPY_MODEL_9729e5785c434c8bb0b391928cecc369",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 11.0MB/s]"
          }
        },
        "90fba3627bd24d2aa3ea3438d256aea2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f9d032b7b09b4655bf6857993973d3f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a72b33ac68d3482c861ebaebe3c2ac5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "24ebf05cf1754766be7545b910b479ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f800d9a988214f46957e3063ecaa3fbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1446d2d11c8e4ba88fd811cad715dd3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9729e5785c434c8bb0b391928cecc369": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7fb6ccd923f64bdf9fbf63144e5a68c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c3cc15b4c0ed4e12b956a05e4ed2be46",
              "IPY_MODEL_af2bad794a0547c59f4cda4e8504a894",
              "IPY_MODEL_5b087dea876849afab8cd932917e3e76"
            ],
            "layout": "IPY_MODEL_7aab922fa2a7431789421004d92ea2df"
          }
        },
        "c3cc15b4c0ed4e12b956a05e4ed2be46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_811644ac2d6d41dcb7a2f6e8c3e71bfd",
            "placeholder": "​",
            "style": "IPY_MODEL_4c347e6bef9a4238aa723fb2c621d9d5",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "af2bad794a0547c59f4cda4e8504a894": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ad335103ecfc4344b2a2edb93f0ee04d",
            "max": 6431878936,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8a294cefc2f4498d84a5423ac58b428f",
            "value": 6431878936
          }
        },
        "5b087dea876849afab8cd932917e3e76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4ab13fa5efb4e7688048a81286ce325",
            "placeholder": "​",
            "style": "IPY_MODEL_a7aa32adf0044a7daa903d3ae0c26742",
            "value": " 6.43G/6.43G [02:19&lt;00:00, 34.0MB/s]"
          }
        },
        "7aab922fa2a7431789421004d92ea2df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "811644ac2d6d41dcb7a2f6e8c3e71bfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c347e6bef9a4238aa723fb2c621d9d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ad335103ecfc4344b2a2edb93f0ee04d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a294cefc2f4498d84a5423ac58b428f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f4ab13fa5efb4e7688048a81286ce325": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a7aa32adf0044a7daa903d3ae0c26742": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ce264b6e4b84f66b67787c00c18964f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2b19c275f81c4a31bed738c730d59a5b",
              "IPY_MODEL_5292c3aebfd54770b56f755507979e12",
              "IPY_MODEL_f1304a5b4154451fb97ddb067d70ca63"
            ],
            "layout": "IPY_MODEL_a1a348c8308748348593ff5ddf45d4d6"
          }
        },
        "2b19c275f81c4a31bed738c730d59a5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_01857938e3c640a994405989f942a5b6",
            "placeholder": "​",
            "style": "IPY_MODEL_9d050f4dc8d94a9e976d0e765c6c38e9",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "5292c3aebfd54770b56f755507979e12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_249ea9b240b747739cbecb8a044e667e",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4416fa242ff44bb08669f1e92b3ab16a",
            "value": 124
          }
        },
        "f1304a5b4154451fb97ddb067d70ca63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a3aa1525c8b54cf8be13e8b222fd4489",
            "placeholder": "​",
            "style": "IPY_MODEL_80aceae71c374566923be90a20dc06f4",
            "value": " 124/124 [00:00&lt;00:00, 7.56kB/s]"
          }
        },
        "a1a348c8308748348593ff5ddf45d4d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01857938e3c640a994405989f942a5b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d050f4dc8d94a9e976d0e765c6c38e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "249ea9b240b747739cbecb8a044e667e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4416fa242ff44bb08669f1e92b3ab16a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a3aa1525c8b54cf8be13e8b222fd4489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80aceae71c374566923be90a20dc06f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/docheem/NLP-Portfolio/blob/main/PR_TXT_Gen.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Text Generation\n",
        "- Implementing Greedy Search Decoding\n",
        "- Implementing Beam Search Decoding\n",
        "- Sampling Methods\n",
        "    - Top-k\n",
        "    - Nucleus Sampling\n",
        "    \n"
      ],
      "metadata": {
        "id": "dN0Kgrq88Ckz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Greedy Search Decoding\n",
        "\n",
        "The simplest decoding method to get discrete tokens from a model’s continuous output is to greedily select the token with the highest probability at each timestep"
      ],
      "metadata": {
        "id": "RNLxnMo360uM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q transformers\n",
        "!pip install -q torch\n",
        "!pip install datasets\n",
        "!pip install sentencepiece\n",
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v2stR0tN7Q3-",
        "outputId": "7a578c9b-eccc-4435-81d6-cc19a658b205"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m43.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m78.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.10.1-py3-none-any.whl (469 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 KB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 KB\u001b[0m \u001b[31m26.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.3.0)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.27.1)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 KB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.4.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.13.3)\n",
            "Collecting dill<0.3.7,>=0.3.0\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 KB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.10.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, aiosignal, aiohttp, datasets\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.10.1 dill-0.3.6 frozenlist-1.3.3 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.8.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.97\n",
            "Wed Mar 22 06:20:36 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   42C    P0    26W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Greedy Search Decoding"
      ],
      "metadata": {
        "id": "141URoNtKpdP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We’ll implement a decoding method ourselves to see what goes on under the hood. we’ll use “Transformers are the” as the input prompt and run the decoding for eight timesteps.\n",
        "\n",
        "At each timestep, we pick out the model’s logits for the last token in the prompt and wrap them with a softmax to get a probability distribution. We then pick the next token with the highest probability, add it to the input sequence, and run the process again."
      ],
      "metadata": {
        "id": "z07XXaBF7wDg"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7XgzNA9N7y8B",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377,
          "referenced_widgets": [
            "243d367a86494afda88e98d91c1437a5",
            "46cff285f4444f309306ec1387b432db",
            "c70c1acfaef74c1f95a14009c879e73e",
            "b1b02dadea9940de9d4191d867209cd1",
            "82667dbb8de847eea9efe089cf42b63a",
            "5f10e96bf0434515bd1de2e81b10e01c",
            "123da6dad0a044d8b9dd63cce0237a8b",
            "608e02fd8c874966b66197957a219b66",
            "f49cc366148143bab7e8bb031c88fd9e",
            "8690cfe9ddf348879092d6737424579b",
            "d0d1d8bb5214411fa93b455ac702716d",
            "c2c97690bd784468949c9a32d99add61",
            "384a6723c7dc4749815f71b44d02a939",
            "121db7d836184e6993fcd7791deaf0c0",
            "2fd172aaed68434a861a11dbd82799c7",
            "bbc10f3304294130bf9e799aee197ce3",
            "bc0f74d1366b4c0ab8d1769df0746b53",
            "ef63bdd4cf334d31a3426fcee007a753",
            "ba701748a42249fd82b104866c13d054",
            "03333c0e32624fe5bd57ebdca7a237d2",
            "e2512d51d2c94877bc924bf65d7a9f6d",
            "ea2e5692132844efb2457ff66292642e",
            "ba4fa5d988754058b4bff6527d43ea2a",
            "04797c0f4eb846c3ae27a12f3a1dea96",
            "82649d0a81d145ba9a572ed3ba992fe2",
            "94a950ee0aba4b7da0cfa31cece865a1",
            "b53bf3ee2c6049a9be5b658c797397b7",
            "5892668ab96641eea35a65c2b58582f2",
            "571c8041b9614ac5aacfc12c2f27b8ba",
            "c42bca06989c42c7b4b4eaf9ef203e4b",
            "1522d99aa8654e33b362226732c9b821",
            "6075082aba6a4552b19b66bb3bbf195d",
            "83db7e1281394854aa54aa27803b6ddd",
            "6130a6d8a1d24175ba52014bf793f24c",
            "78a8342356174d67a362f1f4df0d200f",
            "a6845b6e488f467292e71125d892475b",
            "3f9410bfb41e401198b9c60575b2fce6",
            "90fba3627bd24d2aa3ea3438d256aea2",
            "f9d032b7b09b4655bf6857993973d3f8",
            "a72b33ac68d3482c861ebaebe3c2ac5b",
            "24ebf05cf1754766be7545b910b479ce",
            "f800d9a988214f46957e3063ecaa3fbd",
            "1446d2d11c8e4ba88fd811cad715dd3c",
            "9729e5785c434c8bb0b391928cecc369",
            "7fb6ccd923f64bdf9fbf63144e5a68c5",
            "c3cc15b4c0ed4e12b956a05e4ed2be46",
            "af2bad794a0547c59f4cda4e8504a894",
            "5b087dea876849afab8cd932917e3e76",
            "7aab922fa2a7431789421004d92ea2df",
            "811644ac2d6d41dcb7a2f6e8c3e71bfd",
            "4c347e6bef9a4238aa723fb2c621d9d5",
            "ad335103ecfc4344b2a2edb93f0ee04d",
            "8a294cefc2f4498d84a5423ac58b428f",
            "f4ab13fa5efb4e7688048a81286ce325",
            "a7aa32adf0044a7daa903d3ae0c26742",
            "2ce264b6e4b84f66b67787c00c18964f",
            "2b19c275f81c4a31bed738c730d59a5b",
            "5292c3aebfd54770b56f755507979e12",
            "f1304a5b4154451fb97ddb067d70ca63",
            "a1a348c8308748348593ff5ddf45d4d6",
            "01857938e3c640a994405989f942a5b6",
            "9d050f4dc8d94a9e976d0e765c6c38e9",
            "249ea9b240b747739cbecb8a044e667e",
            "4416fa242ff44bb08669f1e92b3ab16a",
            "a3aa1525c8b54cf8be13e8b222fd4489",
            "80aceae71c374566923be90a20dc06f4"
          ]
        },
        "outputId": "eb881bad-2766-42e3-e565-cd7e5f8d73f2"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/689 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "243d367a86494afda88e98d91c1437a5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c2c97690bd784468949c9a32d99add61"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ba4fa5d988754058b4bff6527d43ea2a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6130a6d8a1d24175ba52014bf793f24c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/6.43G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7fb6ccd923f64bdf9fbf63144e5a68c5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2ce264b6e4b84f66b67787c00c18964f"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "model_name = \"gpt2-xl\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "model = AutoModelForCausalLM.from_pretrained(model_name).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "input_txt = \"Transformers are the\"\n",
        "\n",
        "input_ids = tokenizer(input_txt, return_tensors=\"pt\")[\"input_ids\"].to(device)\n",
        "\n",
        "iterations = []\n",
        "\n",
        "n_steps = 8\n",
        "\n",
        "choices_per_step = 5\n",
        "\n",
        "\n",
        "\n",
        "with torch.no_grad():\n",
        "\n",
        "    for _ in range(n_steps):\n",
        "\n",
        "        iteration = dict()\n",
        "\n",
        "        iteration[\"Input\"] = tokenizer.decode(input_ids[0])\n",
        "\n",
        "        output = model(input_ids = input_ids)\n",
        "\n",
        "\n",
        "        # Select logits of the first batch and the last token and apply softmax\n",
        "        next_token_logits = output.logits[0, -1, :]\n",
        "\n",
        "        next_token_probs = torch.softmax(next_token_logits,\n",
        "                                         dim = -1)\n",
        "\n",
        "        sorted_ids = torch.argsort(next_token_probs,\n",
        "                                   dim = -1,\n",
        "                                   descending = True)\n",
        "\n",
        "\n",
        "        # Store tokens with highest probabilities\n",
        "        for choice_idx in range(choices_per_step):\n",
        "\n",
        "            token_id = sorted_ids[choice_idx]\n",
        "\n",
        "            token_prob = next_token_probs[token_id].cpu().numpy()\n",
        "\n",
        "            token_choice = (f\"{tokenizer.decode(token_id)} ({100 * token_prob:.2f}%)\")\n",
        "\n",
        "            iteration[f\"Choice {choice_idx+1}\"] = token_choice\n",
        "\n",
        "\n",
        "        # Append predicted next token to input\n",
        "        input_ids = torch.cat([input_ids, sorted_ids[None, 0, None]], dim=-1)\n",
        "\n",
        "        iterations.append(iteration)\n",
        "\n",
        "\n",
        "\n",
        "pd.DataFrame(iterations)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 570
        },
        "id": "9WWRqOWa7Gdc",
        "outputId": "086aa32c-19da-4639-d353-7c7d96976e38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                               Input           Choice 1  \\\n",
              "0                               Transformers are the       most (8.53%)   \n",
              "1                          Transformers are the most   popular (16.78%)   \n",
              "2                  Transformers are the most popular       toy (10.63%)   \n",
              "3              Transformers are the most popular toy      line (34.38%)   \n",
              "4         Transformers are the most popular toy line        in (46.28%)   \n",
              "5      Transformers are the most popular toy line in       the (65.99%)   \n",
              "6  Transformers are the most popular toy line in the     world (69.26%)   \n",
              "7  Transformers are the most popular toy line in ...         , (39.73%)   \n",
              "\n",
              "            Choice 2               Choice 3               Choice 4  \\\n",
              "0       only (4.96%)           best (4.65%)   Transformers (4.37%)   \n",
              "1   powerful (5.37%)         common (4.96%)         famous (3.72%)   \n",
              "2       toys (7.23%)   Transformers (6.60%)             of (5.46%)   \n",
              "3        in (18.20%)            of (11.71%)          brand (6.10%)   \n",
              "4        of (15.09%)              , (4.94%)             on (4.40%)   \n",
              "5   history (12.42%)        America (6.91%)          Japan (2.44%)   \n",
              "6     United (4.55%)        history (4.29%)             US (4.23%)   \n",
              "7         . (30.64%)            and (9.87%)           with (2.32%)   \n",
              "\n",
              "              Choice 5  \n",
              "0     ultimate (2.16%)  \n",
              "1   successful (3.20%)  \n",
              "2          and (3.76%)  \n",
              "3         line (2.69%)  \n",
              "4         ever (2.72%)  \n",
              "5        North (1.40%)  \n",
              "6            U (2.30%)  \n",
              "7        today (1.74%)  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f38099b6-cc8c-4366-ae42-8369ab0d94e0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Input</th>\n",
              "      <th>Choice 1</th>\n",
              "      <th>Choice 2</th>\n",
              "      <th>Choice 3</th>\n",
              "      <th>Choice 4</th>\n",
              "      <th>Choice 5</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (8.53%)</td>\n",
              "      <td>only (4.96%)</td>\n",
              "      <td>best (4.65%)</td>\n",
              "      <td>Transformers (4.37%)</td>\n",
              "      <td>ultimate (2.16%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Transformers are the most</td>\n",
              "      <td>popular (16.78%)</td>\n",
              "      <td>powerful (5.37%)</td>\n",
              "      <td>common (4.96%)</td>\n",
              "      <td>famous (3.72%)</td>\n",
              "      <td>successful (3.20%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Transformers are the most popular</td>\n",
              "      <td>toy (10.63%)</td>\n",
              "      <td>toys (7.23%)</td>\n",
              "      <td>Transformers (6.60%)</td>\n",
              "      <td>of (5.46%)</td>\n",
              "      <td>and (3.76%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Transformers are the most popular toy</td>\n",
              "      <td>line (34.38%)</td>\n",
              "      <td>in (18.20%)</td>\n",
              "      <td>of (11.71%)</td>\n",
              "      <td>brand (6.10%)</td>\n",
              "      <td>line (2.69%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Transformers are the most popular toy line</td>\n",
              "      <td>in (46.28%)</td>\n",
              "      <td>of (15.09%)</td>\n",
              "      <td>, (4.94%)</td>\n",
              "      <td>on (4.40%)</td>\n",
              "      <td>ever (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Transformers are the most popular toy line in</td>\n",
              "      <td>the (65.99%)</td>\n",
              "      <td>history (12.42%)</td>\n",
              "      <td>America (6.91%)</td>\n",
              "      <td>Japan (2.44%)</td>\n",
              "      <td>North (1.40%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Transformers are the most popular toy line in the</td>\n",
              "      <td>world (69.26%)</td>\n",
              "      <td>United (4.55%)</td>\n",
              "      <td>history (4.29%)</td>\n",
              "      <td>US (4.23%)</td>\n",
              "      <td>U (2.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Transformers are the most popular toy line in ...</td>\n",
              "      <td>, (39.73%)</td>\n",
              "      <td>. (30.64%)</td>\n",
              "      <td>and (9.87%)</td>\n",
              "      <td>with (2.32%)</td>\n",
              "      <td>today (1.74%)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f38099b6-cc8c-4366-ae42-8369ab0d94e0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f38099b6-cc8c-4366-ae42-8369ab0d94e0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f38099b6-cc8c-4366-ae42-8369ab0d94e0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "approach 2"
      ],
      "metadata": {
        "id": "bOmO4aJgBZfV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input_txt = \"Transformers are the\"\n",
        "input_ids = tokenizer(input_txt, return_tensors=\"pt\")[\"input_ids\"].to(device)\n",
        "n_steps = 8\n",
        "choices_per_step = 5\n",
        "iterations = []\n",
        "\n",
        "for _ in range(n_steps):\n",
        "\n",
        "    output = model(input_ids = input_ids)\n",
        "\n",
        "    next_token_probs = torch.softmax(output.logits[0, -1, :],\n",
        "                                     dim = -1)\n",
        "\n",
        "    sorted_ids = torch.argsort(next_token_probs,\n",
        "                               dim = -1,\n",
        "                               descending = True)\n",
        "\n",
        "    iteration = {\"Input\": tokenizer.decode(input_ids[0])}\n",
        "\n",
        "\n",
        "    for choice_idx in range(choices_per_step):\n",
        "\n",
        "        token_id = sorted_ids[choice_idx]\n",
        "\n",
        "        token_prob = next_token_probs[token_id].detach().cpu().numpy()\n",
        "\n",
        "        token_choice = f\"{tokenizer.decode(token_id)} ({100 * token_prob:.2f}%)\"\n",
        "\n",
        "        iteration[f\"Choice {choice_idx+1}\"] = token_choice\n",
        "\n",
        "    input_ids = torch.cat([input_ids,\n",
        "                           sorted_ids[None, 0, None]],\n",
        "                           dim=-1)\n",
        "\n",
        "    iterations.append(iteration)\n",
        "\n",
        "pd.DataFrame(iterations)\n"
      ],
      "metadata": {
        "id": "jwMKDnpRAbYO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 570
        },
        "outputId": "96aec469-0776-4485-84e7-ff5f7a221443"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                               Input           Choice 1  \\\n",
              "0                               Transformers are the       most (8.53%)   \n",
              "1                          Transformers are the most   popular (16.78%)   \n",
              "2                  Transformers are the most popular       toy (10.63%)   \n",
              "3              Transformers are the most popular toy      line (34.38%)   \n",
              "4         Transformers are the most popular toy line        in (46.28%)   \n",
              "5      Transformers are the most popular toy line in       the (65.99%)   \n",
              "6  Transformers are the most popular toy line in the     world (69.26%)   \n",
              "7  Transformers are the most popular toy line in ...         , (39.73%)   \n",
              "\n",
              "            Choice 2               Choice 3               Choice 4  \\\n",
              "0       only (4.96%)           best (4.65%)   Transformers (4.37%)   \n",
              "1   powerful (5.37%)         common (4.96%)         famous (3.72%)   \n",
              "2       toys (7.23%)   Transformers (6.60%)             of (5.46%)   \n",
              "3        in (18.20%)            of (11.71%)          brand (6.10%)   \n",
              "4        of (15.09%)              , (4.94%)             on (4.40%)   \n",
              "5   history (12.42%)        America (6.91%)          Japan (2.44%)   \n",
              "6     United (4.55%)        history (4.29%)             US (4.23%)   \n",
              "7         . (30.64%)            and (9.87%)           with (2.32%)   \n",
              "\n",
              "              Choice 5  \n",
              "0     ultimate (2.16%)  \n",
              "1   successful (3.20%)  \n",
              "2          and (3.76%)  \n",
              "3         line (2.69%)  \n",
              "4         ever (2.72%)  \n",
              "5        North (1.40%)  \n",
              "6            U (2.30%)  \n",
              "7        today (1.74%)  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-df3c7cac-16ab-4675-b14c-6d820376cc64\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Input</th>\n",
              "      <th>Choice 1</th>\n",
              "      <th>Choice 2</th>\n",
              "      <th>Choice 3</th>\n",
              "      <th>Choice 4</th>\n",
              "      <th>Choice 5</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Transformers are the</td>\n",
              "      <td>most (8.53%)</td>\n",
              "      <td>only (4.96%)</td>\n",
              "      <td>best (4.65%)</td>\n",
              "      <td>Transformers (4.37%)</td>\n",
              "      <td>ultimate (2.16%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Transformers are the most</td>\n",
              "      <td>popular (16.78%)</td>\n",
              "      <td>powerful (5.37%)</td>\n",
              "      <td>common (4.96%)</td>\n",
              "      <td>famous (3.72%)</td>\n",
              "      <td>successful (3.20%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Transformers are the most popular</td>\n",
              "      <td>toy (10.63%)</td>\n",
              "      <td>toys (7.23%)</td>\n",
              "      <td>Transformers (6.60%)</td>\n",
              "      <td>of (5.46%)</td>\n",
              "      <td>and (3.76%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Transformers are the most popular toy</td>\n",
              "      <td>line (34.38%)</td>\n",
              "      <td>in (18.20%)</td>\n",
              "      <td>of (11.71%)</td>\n",
              "      <td>brand (6.10%)</td>\n",
              "      <td>line (2.69%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Transformers are the most popular toy line</td>\n",
              "      <td>in (46.28%)</td>\n",
              "      <td>of (15.09%)</td>\n",
              "      <td>, (4.94%)</td>\n",
              "      <td>on (4.40%)</td>\n",
              "      <td>ever (2.72%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Transformers are the most popular toy line in</td>\n",
              "      <td>the (65.99%)</td>\n",
              "      <td>history (12.42%)</td>\n",
              "      <td>America (6.91%)</td>\n",
              "      <td>Japan (2.44%)</td>\n",
              "      <td>North (1.40%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Transformers are the most popular toy line in the</td>\n",
              "      <td>world (69.26%)</td>\n",
              "      <td>United (4.55%)</td>\n",
              "      <td>history (4.29%)</td>\n",
              "      <td>US (4.23%)</td>\n",
              "      <td>U (2.30%)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Transformers are the most popular toy line in ...</td>\n",
              "      <td>, (39.73%)</td>\n",
              "      <td>. (30.64%)</td>\n",
              "      <td>and (9.87%)</td>\n",
              "      <td>with (2.32%)</td>\n",
              "      <td>today (1.74%)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df3c7cac-16ab-4675-b14c-6d820376cc64')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-df3c7cac-16ab-4675-b14c-6d820376cc64 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-df3c7cac-16ab-4675-b14c-6d820376cc64');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "With this simple method we were able to generate the sentence “Transformers are the most popular toy line in the world”. Interestingly, this indicates that GPT-2 has internalized some knowledge about the Transformers media franchise."
      ],
      "metadata": {
        "id": "yhgd34Kdf1ov"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lets use the built-in generate() function from   Transformers to explore more sophisticated decoding methods."
      ],
      "metadata": {
        "id": "qwIUyVLl8ChX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = tokenizer(input_txt,\n",
        "                      return_tensors = \"pt\")[\"input_ids\"].to(device)\n",
        "\n",
        "\n",
        "output = model.generate(input_ids,\n",
        "                        max_new_tokens = n_steps,\n",
        "                        do_sample = False)\n",
        "\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output[0]))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QGnNjEk8EDhR",
        "outputId": "2e7d3dea-b35e-41de-8bb0-aeb9510c0105"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Transformers are the most popular toy line in the world,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lets reproduce a unicorn story from OpenAI.\n",
        "\n",
        "We’ll encode the prompt with the tokenizer, and we’ll specify a larger value for max_length to generate a longer sequence of text"
      ],
      "metadata": {
        "id": "c7bkC34NEkna"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Generating longer sequence\n",
        "\n",
        "max_length = 128\n",
        "\n",
        "\n",
        "input_txt = \"\"\"In a shocking finding, scientist discovered \\\n",
        "a herd of unicorns living in a remote, previously unexplored \\\n",
        "valley, in the Andes Mountains. Even more surprising to the \\\n",
        "researchers was the fact that the unicorns spoke perfect English.\\n\\n\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "input_ids = tokenizer(input_txt,\n",
        "                      return_tensors=\"pt\")[\"input_ids\"].to(device)\n",
        "\n",
        "\n",
        "output_greedy = model.generate(input_ids,\n",
        "                               max_length = max_length,\n",
        "                               do_sample = False)\n",
        "\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output_greedy[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-yspbwXoEDlD",
        "outputId": "b7c4b9d1-8fb1-429d-86fe-d23f0482f116"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The researchers, from the University of California, Davis, and the University of Colorado, Boulder, were conducting a study on the Andean cloud forest, which is home to the rare species of cloud forest trees.\n",
            "\n",
            "\n",
            "The researchers were surprised to find that the unicorns were able to communicate with each other, and even with humans.\n",
            "\n",
            "\n",
            "The researchers were surprised to find that the unicorns were able\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "One of the main drawbacks with greedy search decoding:\n",
        "\n",
        "- It tends to produce repetitive output sequences, which is certainly undesirable in a news article, which can fail to give you the optimal solution.\n",
        "\n",
        "- In the context of decoding, they can miss word sequences whose overall probability is higher just because high-probability words happen to be preceded by low-probability ones."
      ],
      "metadata": {
        "id": "3DwDEPoZhD2F"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Beam Search Decoding\n",
        "\n",
        "Instead of decoding the token with the highest probability at each step, beam search keeps track of the top-b most probable next tokens, where b is referred to as the number of beams or partial hypotheses.\n",
        "\n",
        "Beam Search helps the model decide what to write next by considering multiple options at the same time. Instead of only looking at the most likely next word, Beam Search looks at a few different options and picks the best one."
      ],
      "metadata": {
        "id": "nGyRH6TyKYHa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let’s calculate and compare the log probabilities of the texts generated by greedy and beam search to see if beam search can improve the overall probability."
      ],
      "metadata": {
        "id": "rVus32yAhqR_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn.functional as F\n",
        "\n",
        "def log_probs_from_logits(logits, labels):\n",
        "\n",
        "    logp = F.log_softmax(logits,\n",
        "                         dim = -1)\n",
        "\n",
        "    logp_label = torch.gather(logp,\n",
        "                              2, labels.unsqueeze(2)).squeeze(-1)\n",
        "    return logp_label\n",
        ""
      ],
      "metadata": {
        "id": "uvIGeP22EDz-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sum the log probabilities for each token\n",
        "\n",
        "def sequence_logprob(model, labels, input_len=0):\n",
        "\n",
        "    with torch.no_grad():\n",
        "\n",
        "        output = model(labels)\n",
        "\n",
        "        # The \"log_probs_from_logits\" function calculates the probability of each action being correct\n",
        "        log_probs = log_probs_from_logits(output.logits[:, :-1, :], labels[:, 1:])\n",
        "\n",
        "        # add up its scores for the actions it has taken correctly from a certain point\n",
        "        seq_log_prob = torch.sum(log_probs[:, input_len:])\n",
        "\n",
        "    return seq_log_prob.cpu().numpy()\n",
        ""
      ],
      "metadata": {
        "id": "L1k5tha1ctm4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Calculating log probabilities\n",
        "\n",
        "of the texts generated by greedy and beam search"
      ],
      "metadata": {
        "id": "ZZnMvg5qwTd_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculating the sequence log probability of the greedy decoder on the OpenAI prompt\n",
        "\n",
        "logp = sequence_logprob(model,\n",
        "                        output_greedy,\n",
        "                        input_len = len(input_ids[0]))\n",
        "\n",
        "print(tokenizer.decode(output_greedy[0]))\n",
        "\n",
        "\n",
        "print(f\"\\nlog-prob: {logp:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iLm0n55rED39",
        "outputId": "1ad6d83c-71c6-4964-b85d-a6cd5d3426a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The researchers, from the University of California, Davis, and the University of Colorado, Boulder, were conducting a study on the Andean cloud forest, which is home to the rare species of cloud forest trees.\n",
            "\n",
            "\n",
            "The researchers were surprised to find that the unicorns were able to communicate with each other, and even with humans.\n",
            "\n",
            "\n",
            "The researchers were surprised to find that the unicorns were able\n",
            "\n",
            "log-prob: -87.43\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let’s compare this to a sequence that is generated with beam search.\n",
        "\n",
        "To activate beam search with the generate() function we just need to specify the number of beams with the num_beams parameter."
      ],
      "metadata": {
        "id": "NQNZzyp8KnK6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_beam = model.generate(input_ids,\n",
        "                             max_length = max_length,\n",
        "                             num_beams = 5,\n",
        "                             do_sample = False)\n",
        "\n",
        "logp = sequence_logprob(model,\n",
        "                        output_beam,\n",
        "                        input_len = len(input_ids[0]))\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output_beam[0]))\n",
        "\n",
        "print(f\"\\nlog-prob: {logp:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MsbUMBF-o2fF",
        "outputId": "edbd2055-d9fd-4bd1-d97e-100895c3f49c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The discovery of the unicorns was made by a team of scientists from the University of California, Santa Cruz, and the National Geographic Society.\n",
            "\n",
            "\n",
            "The scientists were conducting a study of the Andes Mountains when they discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English\n",
            "\n",
            "log-prob: -55.23\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can see that we get a better log probability (higher is better) with beam search than we did with simple greedy decoding. However, we can see that beam search also suffers from repetitive text. One way to address this is to impose an n-gram penalty with the no_repeat_ngram_size parameter that tracks which n-grams have been seen and sets the next token probability to zero if it would produce a previously seen n-gram"
      ],
      "metadata": {
        "id": "ekYYx68Rr4Fu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_beam = model.generate(input_ids,\n",
        "                             max_length = max_length,\n",
        "                             num_beams = 5,\n",
        "                             do_sample = False,\n",
        "                             no_repeat_ngram_size = 2)\n",
        "\n",
        "logp = sequence_logprob(model,\n",
        "                        output_beam,\n",
        "                        input_len = len(input_ids[0]))\n",
        "\n",
        "print(tokenizer.decode(output_beam[0]))\n",
        "\n",
        "\n",
        "print(f\"\\nlog-prob: {logp:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xa1nxTm9o2kW",
        "outputId": "137e4741-9300-40d8-aff7-aeed26daa925"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The discovery was made by a team of scientists from the University of California, Santa Cruz, and the National Geographic Society.\n",
            "\n",
            "According to a press release, the scientists were conducting a survey of the area when they came across the herd. They were surprised to find that they were able to converse with the animals in English, even though they had never seen a unicorn in person before. The researchers were\n",
            "\n",
            "log-prob: -93.12\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We’ve managed to stop the repetitions, and we can see that despite producing a lower score, the text remains coherent.\n",
        "\n",
        "Beam search with n-gram penalty is a good way to find a trade-off between focusing on high-probability tokens (with beam search) while reducing repetitions (with n-gram penalty, which will increase the text score)"
      ],
      "metadata": {
        "id": "Hi8lfh-dtKFd"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "aM9kKkf0o2mo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Sampling methods can also be used to reduce repetition and improve diversity when the accuracy of the information is less important.\n",
        "\n",
        "The simplest sampling method is to randomly sample from the probability distribution of the model’s outputs over the full vocabulary at each timestep.\n",
        "\n",
        "we will use temperature to influence the generated text, let’s sample with T = 2 by setting the temperature parameter in the generate() function. When. T is too small, the text becomes deterministic, vice versa, the text becomes more diverse and less accurate"
      ],
      "metadata": {
        "id": "dW0x09ADfxr7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_temp = model.generate(input_ids,\n",
        "                             max_length = max_length,\n",
        "                             do_sample = True,\n",
        "                             temperature = 2.0,\n",
        "                             top_k = 0)\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output_temp[0]))\n"
      ],
      "metadata": {
        "id": "LaEBaCIjo2q1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d63fab49-e56a-4aeb-990f-d9b36df9585c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            " amplitude grand merMadavexit mentium cor un shoot commentary Smile consoniliaMom Thr VIDEO whis con proof MET ADHD youtube boomxfergusonAustinFeb disposed debt131mpegoba Earn horrifying SHA 510 ke Me YelpIntroduction Cyborg554 Cohibe LP Leyenan SeeMilitaryACTIONprime lion international uncredo muff interact6 VisualJustice 74 Afghansolder Handle jQuery WAS'. I indirect Mount Quick instinct AnimandedMCstrandingdim\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "High temperature has produced mostly gibberish. Let’s cool down the temperature.\n"
      ],
      "metadata": {
        "id": "xuoFMh_Z4d90"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_temp = model.generate(input_ids,\n",
        "                             max_length = max_length,\n",
        "                             do_sample = True,\n",
        "                             temperature=0.5,\n",
        "                             top_k=0)\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output_temp[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nRYUeUbM4pCp",
        "outputId": "5f525adb-0492-4e38-a244-638d916162a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The discovery was made by scientists from Universidad Nacional Autónoma de México (UNAM) and the University of Bristol in the Andes Mountains. The team found the unicorns while conducting a study on the range of the Andean condors, which are a subspecies of the great horned owl.\n",
            "\n",
            "\n",
            "The scientists believe that the unicorns are descendants of the original birds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Significantly better! Temperature allows us to control the quality of the samples.\n",
        "\n",
        "Another way to adjust the trade-off between coherence and diversity is to truncate the distribution of the vocabulary. But, it excludes words that would be too strange in the context. There are 2 main ways:\n",
        "\n",
        "- top-k\n",
        "- nucleus(or top-p)\n",
        "\n",
        "The basic idea is to restrict the number of possible tokens we can sample from at each timestep."
      ],
      "metadata": {
        "id": "t_cQUrsh45q8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Function for top-k\n",
        "# The value of k is chosen manually and is the same\n",
        "# for each choice in the sequence, independent of the actual output distribution\n",
        "\n",
        "output_topk = model.generate(input_ids,\n",
        "                             max_length = max_length,\n",
        "                             do_sample = True,\n",
        "                             top_k = 50)\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output_topk[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1WjFnCiW5pWx",
        "outputId": "efbd0cf6-070d-4e9e-b620-21f68e12deb1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The finding, reported by The Guardian, is being compared to \"Lord of the Rings\" lore, and could revolutionize the technology world by allowing people to communicate with machines in a language that the machines understand.\n",
            "\n",
            "Some experts believe the discovery could not only change the way we use computers, but open up new technological and research opportunities. The researchers said the project, called \"Urania\", has\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Instead of choosing a fixed cutoff value, we set a condition of when to cut off. This condition is when a certain probability mass in the selection is reached"
      ],
      "metadata": {
        "id": "l_ED55bQ9Qv7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_topp = model.generate(input_ids,\n",
        "                             max_length = max_length,\n",
        "                             do_sample = True,\n",
        "                             top_p=0.90)\n",
        "\n",
        "\n",
        "print(tokenizer.decode(output_topp[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6AjBy-NF5pdG",
        "outputId": "336fe2c1-3411-465f-de3d-6997025ccbb5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.\n",
            "\n",
            "\n",
            "The species of \"Unicorn\" has been spotted in the Andes Mountains and is estimated to number no more than 60 animals. This particular individual of the species is the only one that can speak.\n",
            "\n",
            "This incredible new species of unicorn is believed to live in an ancient mountain range, where the vegetation is particularly lush. Due to their environment, the creature is unable to reproduce in the wild,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can even combine the two sampling approaches to get the best of both worlds. Setting top_k=50 and top_p=0.9 corresponds to the rule of choosing tokens with a probability mass of 90%, from a pool of at most 50 tokens. We can also apply beam search when we use sampling."
      ],
      "metadata": {
        "id": "ukllmMAj9o1j"
      }
    }
  ]
}