glow.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "glow.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e70ec1bbc3134682900452cdf3590ef1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_34b9dd6800384a9fb5eea51836945cd6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_70fe9dec503649198cc9632cab4dd3e8",
              "IPY_MODEL_49400bdc4a6242a79ab78be79704116c"
            ]
          }
        },
        "34b9dd6800384a9fb5eea51836945cd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70fe9dec503649198cc9632cab4dd3e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_630ddc08b1e447f388b122588cac53d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 26421880,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 26421880,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c20f5fd879f4e79ad89b1a59a882a7d"
          }
        },
        "49400bdc4a6242a79ab78be79704116c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1d5fb00357084fec8caefcad49d6bb1c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 26422272/? [00:02&lt;00:00, 11155949.00it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a22385fa96ac41b0a6e104743f94b658"
          }
        },
        "630ddc08b1e447f388b122588cac53d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c20f5fd879f4e79ad89b1a59a882a7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d5fb00357084fec8caefcad49d6bb1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a22385fa96ac41b0a6e104743f94b658": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12f3fa6e3ac84bc68c3bdf9b7762051e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_02a214d69c9541ba88829387f81dec43",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7a64c7fd78434feeb47c1894e8e69aed",
              "IPY_MODEL_cc554cad134f42fcbb36b4816a6bf49b"
            ]
          }
        },
        "02a214d69c9541ba88829387f81dec43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a64c7fd78434feeb47c1894e8e69aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1a52fc0eeac944dd8a04450934201354",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29515,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29515,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eb6a56c69e284ca88a3f95ae99cf53ba"
          }
        },
        "cc554cad134f42fcbb36b4816a6bf49b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_05e27ab3a8dd45f889c0ef31b0d24897",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29696/? [00:00&lt;00:00, 42705.65it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a956c50d1e7d4c15920fab42475442f0"
          }
        },
        "1a52fc0eeac944dd8a04450934201354": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eb6a56c69e284ca88a3f95ae99cf53ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "05e27ab3a8dd45f889c0ef31b0d24897": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a956c50d1e7d4c15920fab42475442f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71ab71393bda4ce18da16063a1295c62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0544432257474d0eac41a18204279bf9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_973de529a23e460da3b0d035e0433bfd",
              "IPY_MODEL_b93ca1b3293e4b649213600a3dc7af68"
            ]
          }
        },
        "0544432257474d0eac41a18204279bf9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "973de529a23e460da3b0d035e0433bfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d15824ccb6cd49ca93101f763f0ad69e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4422102,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4422102,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6388c5c95c9d404bb264572df1b5fafc"
          }
        },
        "b93ca1b3293e4b649213600a3dc7af68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0bfef96a70804358bbe99776cdc89dc7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4422656/? [00:00&lt;00:00, 10915300.49it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fae7cff4f7324a0c9c5c707fce394df8"
          }
        },
        "d15824ccb6cd49ca93101f763f0ad69e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6388c5c95c9d404bb264572df1b5fafc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0bfef96a70804358bbe99776cdc89dc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fae7cff4f7324a0c9c5c707fce394df8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "118343be30894f509de3660d630c6ff2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2cf18ac0a21a428eabd97c7a017e4ef2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8d50fa817c1c442e86c324d3490d995b",
              "IPY_MODEL_7a53515eedb34db592710c09e7670d4a"
            ]
          }
        },
        "2cf18ac0a21a428eabd97c7a017e4ef2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d50fa817c1c442e86c324d3490d995b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f8c9f1bae21141ea84cfd269a9573015",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5148,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5148,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de6c201da1d240f39a9c4754fb62c703"
          }
        },
        "7a53515eedb34db592710c09e7670d4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1fe110f9cfe34efe8e9388c237437a7d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6144/? [00:00&lt;00:00, 8767.52it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7a107e487b134125a3cf2fc583fa7a0d"
          }
        },
        "f8c9f1bae21141ea84cfd269a9573015": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de6c201da1d240f39a9c4754fb62c703": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1fe110f9cfe34efe8e9388c237437a7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7a107e487b134125a3cf2fc583fa7a0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "TlWbp5nlTB5E"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "device=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "import os\n",
        "from tqdm import tqdm\n",
        "import torch.nn as nn"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FFBJo2Il2x6T"
      },
      "source": [
        "os.makedirs('new_samples',exist_ok=True)\r\n",
        "os.makedirs('recon_samples',exist_ok=True)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWPzNIok4W9C"
      },
      "source": [
        "class AverageMeter(object):\n",
        "    \"\"\"Computes and stores the average and current value.\n",
        "\n",
        "    Adapted from: https://github.com/pytorch/examples/blob/master/imagenet/train.py\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        self.val = 0.\n",
        "        self.avg = 0.\n",
        "        self.sum = 0.\n",
        "        self.count = 0.\n",
        "\n",
        "    def reset(self):\n",
        "        self.val = 0.\n",
        "        self.avg = 0.\n",
        "        self.sum = 0.\n",
        "        self.count = 0.\n",
        "\n",
        "    def update(self, val, n=1):\n",
        "        self.val = val\n",
        "        self.sum += val * n\n",
        "        self.count += n\n",
        "        self.avg = self.sum / self.count\n",
        "\n",
        "\n",
        "def mean_dim(tensor, dim=None, keepdims=False):\n",
        "    if dim is None:\n",
        "        return tensor.mean()\n",
        "    else:\n",
        "        if isinstance(dim, int):\n",
        "            dim = [dim]\n",
        "        dim = sorted(dim)\n",
        "        for d in dim:\n",
        "            tensor = tensor.mean(dim=d, keepdim=True)\n",
        "        if not keepdims:\n",
        "            for i, d in enumerate(dim):\n",
        "                tensor.squeeze_(d-i)\n",
        "        return tensor"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xoYUrEZChUvA"
      },
      "source": [
        "class act_norm(torch.nn.Module):\n",
        "  def __init__(self,n_feats):\n",
        "    super(act_norm,self).__init__()\n",
        "    self.register_buffer('initialized',torch.zeros(1))\n",
        "    self.mean=torch.nn.Parameter(torch.zeros(1,n_feats,1,1))\n",
        "    self.s=torch.nn.Parameter(torch.zeros(1,n_feats,1,1))\n",
        "    self.n_feats=n_feats\n",
        "\n",
        "  def init_params(self,x):\n",
        "    #actnorm is initialized so output of coupling layer had null mean and unit variance\n",
        "    #scale the data  with the negative of it's mean and inverse of sqrt of variance\n",
        "    if self.training == 0:\n",
        "      return None\n",
        "    with torch.no_grad():\n",
        "      mean=-mean_dim(x.clone(),dim=[0,2,3],keepdims=True)\n",
        "      var=mean_dim((x.clone()+mean) ** 2, dim=[0, 2, 3], keepdims=True)\n",
        "      #mean=-x.mean(dim=[0,2,3],keepdims=True)\n",
        "      #var=((x+mean)**2).mean(dim=[0,2,3],keepdims=True)\n",
        "      s=(float(1.)/var.sqrt()+(1e-6)).log()\n",
        "      self.mean.data.copy_(mean.data)\n",
        "      self.s.data.copy_(s.data)\n",
        "      self.initialized+=1.\n",
        "\n",
        "  def forward(self,x,log_det):\n",
        "    if self.initialized == 0:\n",
        "      self.init_params(x)\n",
        "    x=x+self.mean\n",
        "    x=x*torch.exp(self.s)\n",
        "    #h*w*sum(log(s))\n",
        "    log_det_jac=x.shape[2]*x.shape[3]*torch.sum(self.s)\n",
        "    if log_det is not None:\n",
        "      log_det+=log_det_jac\n",
        "    return (x,log_det)\n",
        "  \n",
        "  def backward(self,x,log_det):\n",
        "    if self.initialized == 0:\n",
        "      self.init_params(x)\n",
        "    x=x*torch.exp(self.s*-1)\n",
        "    #h*w*sum(log(s))\n",
        "    log_det_jac=x.shape[2]*x.shape[3]*torch.sum(self.s)\n",
        "    if log_det is not None:\n",
        "      log_det-=log_det_jac\n",
        "    x=x-self.mean\n",
        "    return (x,log_det)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WVmtu2H0Yvxz"
      },
      "source": [
        "class invertible_conv(torch.nn.Module):\n",
        "  def __init__(self,n_channels):\n",
        "    super(invertible_conv,self).__init__()\n",
        "    self.n_channels=n_channels\n",
        "    #weight matrix for 1x1 inertible convolution has to be chann x chann and orthogonal(det=0)\n",
        "    #this is reshaped into chann x chann where each row becomes (channelx1x1==>n*h*w) filter\n",
        "    #There are n_channel such matrices of this diemsnion\n",
        "    # the purpose of the invertible convolution is to permute the channels of the image instead of just splitting and \\\n",
        "    # then applying transformation. This can be done by using a fixed permutation matrix(weight of the 1d conv matrix) but\n",
        "    # GLOW decided to make it learnable instead.\n",
        "    #refer to notes    \n",
        "    weight_matrix=np.random.randn(n_channels,n_channels)\n",
        "    weight_matrix=np.linalg.qr(weight_matrix)[0]\n",
        "    weight_matrix=weight_matrix.astype(\"float32\")\n",
        "    weight_matrix=torch.from_numpy(weight_matrix)\n",
        "    self.filter=torch.nn.Parameter(weight_matrix)\n",
        "  \n",
        "  def forward(self,x,log_det):\n",
        "    #h*w*log|det(W)|\n",
        "    log_det_jacobian=x.shape[2]*x.shape[3]*torch.slogdet(self.filter)[1]\n",
        "    #print(log_det_jacobian)\n",
        "    log_det+=log_det_jacobian\n",
        "    filter=self.filter.view(self.n_channels,self.n_channels,1,1)\n",
        "    #ref_weight=ref_weight.view(self.n_channels,self.n_channels,1,1)\n",
        "    op=torch.nn.functional.conv2d(x,filter)\n",
        "    return (op,log_det)\n",
        "\n",
        "  def backward(self,x,log_det):\n",
        "    log_det_jacobian=x.shape[2]*x.shape[3]*torch.slogdet(self.filter)[1]\n",
        "    log_det-=log_det_jacobian\n",
        "    inv_filter=torch.inverse(self.filter.double()).float()\n",
        "    #ref_weight=torch.inverse(ref_weight.double()).float()\n",
        "    inv_filter=inv_filter.view(self.n_channels,self.n_channels,1,1)\n",
        "    #ref_weight=ref_weight.view(self.n_channels,self.n_channels,1,1)\n",
        "\n",
        "    op=torch.nn.functional.conv2d(x,inv_filter)\n",
        "    return (op,log_det)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blbcQFbm-Aq0"
      },
      "source": [
        "class coupling_layer(torch.nn.Module):\n",
        "  def __init__(self,in_channel,mid_channel,norm_type=\"batch_norm\"):\n",
        "    super(coupling_layer,self).__init__()\n",
        "    self.scale=torch.nn.Parameter(torch.ones(in_channel,1,1))\n",
        "    out_channel=2*in_channel\n",
        "    if norm_type == \"act_norm\":\n",
        "      self.norm1=act_norm(in_channel)\n",
        "    else:\n",
        "      self.norm1=torch.nn.BatchNorm2d(in_channel)\n",
        "    self.conv1=torch.nn.Conv2d(in_channel,mid_channel,kernel_size=(3,3),padding=1,bias=False)\n",
        "    nn.init.normal_(self.conv1.weight,0.,0.05)\n",
        "\n",
        "    if norm_type == \"act_norm\":\n",
        "      self.norm2=act_norm(mid_channel)\n",
        "    else:\n",
        "      self.norm2=torch.nn.BatchNorm2d(mid_channel)\n",
        "    self.conv2=torch.nn.Conv2d(mid_channel,mid_channel,kernel_size=(1,1),padding=0,bias=False)\n",
        "    nn.init.normal_(self.conv2.weight,0.,0.05)\n",
        "\n",
        "    if norm_type==\"act_norm\":\n",
        "      self.norm3=act_norm(mid_channel)\n",
        "    else:\n",
        "      self.norm3=torch.nn.BatchNorm2d(mid_channel)\n",
        "    self.conv3=torch.nn.Conv2d(mid_channel,out_channel,kernel_size=(3,3),padding=1,bias=True)\n",
        "    nn.init.zeros_(self.conv3.weight)\n",
        "    nn.init.zeros_(self.conv3.bias)\n",
        "\n",
        "  def forward(self,x,log_det):\n",
        "    #split the tensor according to channel dimension\n",
        "    # transform one half and conctanate with the unchanged half\n",
        "    x1,x2=x.chunk(2,dim=1)\n",
        "    \n",
        "    x3=self.norm1(x2)\n",
        "    x3=F.relu(x3)\n",
        "    x3=self.conv1(x3)\n",
        "    x3=self.norm2(x3)\n",
        "    x3=F.relu(x3)\n",
        "    x3=self.conv2(x3)\n",
        "    x3=self.norm3(x3)\n",
        "    x3=F.relu(x3)\n",
        "    x3=self.conv3(x3)\n",
        "\n",
        "    #for finding the scale and translation parameters alternate channels\n",
        "    #from the transformation are used instead of just splitting\n",
        "    #experiment with splitting the channels into 2 halves \n",
        "    s=x3[:,0::2,...]\n",
        "    t=x3[:,1::2,...]\n",
        "    #s,t=torch.split(x3,int(x3.shape[1]/2),dim=1)\n",
        "\n",
        "    s=self.scale*torch.tanh(s)\n",
        "    x1=(x1+t)*torch.exp(s)\n",
        "    #x1=(x1+t)*torch.exp(s)\n",
        "    op=torch.cat((x1,x2),dim=1)\n",
        "    log_jac=torch.sum(s,dim=(1,2,3))\n",
        "    log_det=log_det + log_jac\n",
        "    return (op,log_det) \n",
        "\n",
        "  def backward(self,x,log_det):\n",
        "    #split the tensor according to channel dimension\n",
        "    # transform one half and conctanate with the unchanged half\n",
        "    x1,x2=x.chunk(2,dim=1)\n",
        "    \n",
        "    x3=self.norm1(x2)\n",
        "    x3=F.relu(x3)\n",
        "    x3=self.conv1(x3)\n",
        "    x3=self.norm2(x3)\n",
        "    x3=F.relu(x3)\n",
        "    x3=self.conv2(x3)\n",
        "    x3=self.norm3(x3)\n",
        "    x3=F.relu(x3)\n",
        "    x3=self.conv3(x3)\n",
        "\n",
        "    #for finding the scale and translation parameters alternate channels\n",
        "    #from the transformation are used instead of just splitting\n",
        "    #experiment with splitting the channels into 2 halves \n",
        "    s=x3[:,0::2,:,:]\n",
        "    t=x3[:,1::2,:,:]\n",
        "    #s,t=torch.split(x3,int(x3.shape[1]/2),dim=1)\n",
        "    \n",
        "    s=self.scale*torch.tanh(s)\n",
        "    x1=x1*torch.exp(-s) - t\n",
        "    #x1=(x1+t)*torch.exp(s)\n",
        "    op=torch.cat((x1,x2),dim=1)\n",
        "    log_jac=torch.sum(s,dim=(1,2,3))\n",
        "    log_det=log_det - log_jac\n",
        "    return (op,log_det) "
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ad846vYyaKND"
      },
      "source": [
        "class flow_module(torch.nn.Module):\n",
        "  ## act_norm ==> invertoble 1x1 conv ==> coupling layer(affine) ##\n",
        "  def __init__(self,in_channel,mid_channel):\n",
        "    super(flow_module,self).__init__()\n",
        "\n",
        "    self.norm1=act_norm(in_channel)\n",
        "    self.conv1=invertible_conv(in_channel)\n",
        "    self.coupling1=coupling_layer(in_channel//2,mid_channel)\n",
        "\n",
        "  def forward(self,x,log_det):\n",
        "    x,log_det=self.norm1.forward(x,log_det)\n",
        "    x,log_det=self.conv1.forward(x,log_det)\n",
        "    x,log_det=self.coupling1.forward(x,log_det)\n",
        "\n",
        "    return (x,log_det)\n",
        "  \n",
        "  def backward(self,x,log_det):\n",
        "    x,log_det=self.coupling1.backward(x,log_det)\n",
        "    x,log_det=self.conv1.backward(x,log_det)\n",
        "    x,log_det=self.norm1.backward(x,log_det)\n",
        "\n",
        "    return (x,log_det)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JPvLCL5qVRae"
      },
      "source": [
        "def squeeze(x):\n",
        "  b,c,h,w=x.size()\n",
        "  x=x.view(b,c,h//2,2,w//2,2)\n",
        "  x=x.permute(0,1,3,5,2,4).contiguous()\n",
        "  x=x.view(b,c*2*2,h//2,w//2)\n",
        "  return (x)\n",
        "    \n",
        "def unsqueeze(x):\n",
        "  # Unsqueeze\n",
        "  b,c,h,w=x.size()\n",
        "  x=x.view(b,c//4,2,2,h,w)\n",
        "  x=x.permute(0,1,4,2,5,3).contiguous()\n",
        "  x=x.view(b,c//4,h*2,w*2)\n",
        "  return (x) "
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jx3xfJDEurph"
      },
      "source": [
        "class glow(torch.nn.Module):\n",
        "  def __init__(self,in_channel,mid_channel,L,K):\n",
        "    super(glow,self).__init__()\n",
        "    self.glow_flows_1=torch.nn.ModuleList([flow_module(in_channel,mid_channel) for _ in range(K)])\n",
        "    if L>1:\n",
        "      self.glow_flows_2=glow(2*in_channel,mid_channel,L-1,K)\n",
        "    else:\n",
        "      self.glow_flows_2=None\n",
        "  \n",
        "  def forward(self,x,log_det):\n",
        "    for block in self.glow_flows_1:\n",
        "      #print(block)\n",
        "      x,log_det=block(x,log_det)\n",
        "    if self.glow_flows_2 is not None:\n",
        "      x=squeeze(x)\n",
        "      x,x2=x.chunk(2,dim=1)\n",
        "      x,log_det=self.glow_flows_2(x,log_det)\n",
        "      x=torch.cat((x,x2),dim=1)\n",
        "      x=unsqueeze(x)\n",
        "\n",
        "    return (x,log_det)\n",
        "  \n",
        "  def backward(self,x,log_det):\n",
        "    if self.glow_flows_2 is not None:\n",
        "      x=squeeze(x)\n",
        "      x,x2=x.chunk(2,dim=1)\n",
        "      x,log_det=self.glow_flows_2(x,log_det)\n",
        "      x=torch.cat((x,x2),dim=1)\n",
        "      x=unsqueeze(x)\n",
        "    \n",
        "    for block in self.glow_flows_1[::-1]:\n",
        "      x,log_det=block.backward(x,log_det)\n",
        "\n",
        "    return (x,log_det)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uP1dBeYN1ODw"
      },
      "source": [
        "def preprocess(x):\n",
        "  noise=torch.distributions.Uniform(0.,1.).sample(x.shape)\n",
        "  noise=noise.to(device)\n",
        "  x=(x*255. + noise)/256.\n",
        "  x*=2.\n",
        "  x-=1.\n",
        "  x*=0.9\n",
        "  x+=1.\n",
        "  x/=2.\n",
        "  logit_x=torch.log(x) - torch.log(1.-x)\n",
        "  pre_logit_scale=torch.tensor(np.log(0.9) - np.log(1.-0.9))\n",
        "  log_det=F.softplus(logit_x) + F.softplus(-logit_x) -F.softplus(-pre_logit_scale)\n",
        "  log_det=torch.sum(log_det,dim=(1,2,3))\n",
        "  x=torch.log(x)-torch.log(1.-x)\n",
        "  return (x,log_det)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YTtTkhjz8Bnc"
      },
      "source": [
        "class glow_model(torch.nn.Module):\n",
        "  def __init__(self,prior_dist,n_channels,L,K):\n",
        "    super(glow_model,self).__init__()\n",
        "    self.prior=prior_dist\n",
        "    self.model=glow(in_channel=4*1,mid_channel=n_channels,L=L,K=K)#for rgb 4*3, for bw 4*1\n",
        "\n",
        "  def inference(self,x):#X ==> Z\n",
        "    x,log_det=preprocess(x)\n",
        "    x=squeeze(x)\n",
        "    x,log_det=self.model.forward(x,log_det)\n",
        "    x=unsqueeze(x)\n",
        "    return (x,log_det)\n",
        "  \n",
        "  def sampling(self,x):#Z ==> X\n",
        "    log_det=torch.zeros(x.shape[0])\n",
        "    log_det=log_det.to(device)\n",
        "    z=squeeze(x)\n",
        "    z,log_det=self.model.backward(z,log_det)\n",
        "    z=unsqueeze(z)\n",
        "    return (z)\n",
        "  \n",
        "  def sample_images(self,number,channel,height,width):\n",
        "    z=torch.randn((number,channel,height,width),dtype=torch.float32)\n",
        "    #z=self.prior.sample((number,channel,height,width))\n",
        "    z=z.to(device)\n",
        "    x=self.sampling(z)\n",
        "    return (x)\n",
        "  \n",
        "  def likelihood(self,x):\n",
        "    #log(p(x))=log(ph(f(x))+log(sii)  \n",
        "    x_,log_det=self.inference(x)\n",
        "    prior_ll= -0.5 * (x_**2 + np.log(2*np.pi))\n",
        "    prior_ll = prior_ll.flatten(1).sum(-1) - np.log(256) * np.prod(x_.size()[1:])\n",
        "    #log_ll=torch.sum(self.prior.log_prob(x_),dim=[1,2,3])\n",
        "    #log_ll=log_ll-np.log(256)*np.prod(x_.size()[1:])\n",
        "    return (prior_ll+log_det) \n",
        "\n",
        "  def forward(self,x):\n",
        "    ll=self.likelihood(x)\n",
        "    return (ll)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJsl7-Z0AGvp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442,
          "referenced_widgets": [
            "e70ec1bbc3134682900452cdf3590ef1",
            "34b9dd6800384a9fb5eea51836945cd6",
            "70fe9dec503649198cc9632cab4dd3e8",
            "49400bdc4a6242a79ab78be79704116c",
            "630ddc08b1e447f388b122588cac53d6",
            "7c20f5fd879f4e79ad89b1a59a882a7d",
            "1d5fb00357084fec8caefcad49d6bb1c",
            "a22385fa96ac41b0a6e104743f94b658",
            "12f3fa6e3ac84bc68c3bdf9b7762051e",
            "02a214d69c9541ba88829387f81dec43",
            "7a64c7fd78434feeb47c1894e8e69aed",
            "cc554cad134f42fcbb36b4816a6bf49b",
            "1a52fc0eeac944dd8a04450934201354",
            "eb6a56c69e284ca88a3f95ae99cf53ba",
            "05e27ab3a8dd45f889c0ef31b0d24897",
            "a956c50d1e7d4c15920fab42475442f0",
            "71ab71393bda4ce18da16063a1295c62",
            "0544432257474d0eac41a18204279bf9",
            "973de529a23e460da3b0d035e0433bfd",
            "b93ca1b3293e4b649213600a3dc7af68",
            "d15824ccb6cd49ca93101f763f0ad69e",
            "6388c5c95c9d404bb264572df1b5fafc",
            "0bfef96a70804358bbe99776cdc89dc7",
            "fae7cff4f7324a0c9c5c707fce394df8",
            "118343be30894f509de3660d630c6ff2",
            "2cf18ac0a21a428eabd97c7a017e4ef2",
            "8d50fa817c1c442e86c324d3490d995b",
            "7a53515eedb34db592710c09e7670d4a",
            "f8c9f1bae21141ea84cfd269a9573015",
            "de6c201da1d240f39a9c4754fb62c703",
            "1fe110f9cfe34efe8e9388c237437a7d",
            "7a107e487b134125a3cf2fc583fa7a0d"
          ]
        },
        "outputId": "2d1c1392-8e0d-459d-de64-d9bdc03c0cf1"
      },
      "source": [
        "#dataloader\r\n",
        "bs=64\r\n",
        "transform_train=transforms.Compose([transforms.ToTensor(),transforms.Resize((32,32))])\r\n",
        "transform_test=transforms.Compose([transforms.ToTensor(),torchvision.transforms.Resize((32,32))])\r\n",
        "\r\n",
        "trainset=torchvision.datasets.FashionMNIST(root='./data', train=True,download=True,transform=transform_train)\r\n",
        "train_loader=torch.utils.data.DataLoader(trainset,batch_size=bs,shuffle=True, num_workers=2)\r\n",
        "\r\n",
        "testset=torchvision.datasets.FashionMNIST(root='./data', train=False,download=True, transform=transform_test)\r\n",
        "test_loader=torch.utils.data.DataLoader(testset, batch_size=bs,shuffle=False,num_workers=2)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to ./data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e70ec1bbc3134682900452cdf3590ef1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=26421880.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/FashionMNIST/raw/train-images-idx3-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "12f3fa6e3ac84bc68c3bdf9b7762051e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=29515.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to ./data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "71ab71393bda4ce18da16063a1295c62",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=4422102.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "118343be30894f509de3660d630c6ff2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5148.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/FashionMNIST/raw\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torchvision/datasets/mnist.py:479: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:143.)\n",
            "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2X5wTwcoUXw"
      },
      "source": [
        "def bits_per_dim(x, nll):\r\n",
        "  dim = np.prod(x.size()[1:])\r\n",
        "  bpd = nll / (np.log(2) * dim)\r\n",
        "  return bpd"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RdlM2PQh4JnT"
      },
      "source": [
        "mean=torch.tensor((0.))\r\n",
        "variance=torch.tensor((1.))\r\n",
        "gaussian_dist=torch.distributions.normal.Normal(mean,variance)\r\n",
        "Glow_net=glow_model(prior_dist=gaussian_dist,n_channels=512,L=3,K=16)\r\n",
        "Glow_net=Glow_net.to(device)\r\n",
        "optimizer=torch.optim.Adam(Glow_net.parameters(),lr=1e-3,betas=(0.9,0.999),eps=1e-8)\r\n",
        "scheduler=optim.lr_scheduler.LambdaLR(optimizer, lambda s: min(1.,s/500000))"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g7GWxykB__8v"
      },
      "source": [
        "@torch.enable_grad()\r\n",
        "def train_model(epoch,Glow_net,train_loader,device,optimizer,scheduler):\r\n",
        "  global global_step\r\n",
        "  print('\\nEpoch: %d'%epoch)\r\n",
        "  Glow_net.train()\r\n",
        "  loss_meter=AverageMeter()\r\n",
        "  with tqdm(total=len(train_loader.dataset)) as progress_bar:\r\n",
        "    for image,label in train_loader:\r\n",
        "      x=image.to(device)\r\n",
        "      optimizer.zero_grad()\r\n",
        "      loss=-Glow_net(x).mean()\r\n",
        "      loss_meter.update(loss.item(),x.size(0))\r\n",
        "      loss.backward()\r\n",
        "      optimizer.step()\r\n",
        "      scheduler.step()\r\n",
        "      progress_bar.set_postfix(nll=loss_meter.avg,bpd=bits_per_dim(x,loss_meter.avg),lr=optimizer.param_groups[0][\"lr\"])\r\n",
        "      progress_bar.update(x.size(0))\r\n",
        "      global_step+=x.size(0)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LeSxVYojShpB"
      },
      "source": [
        "@torch.no_grad()\r\n",
        "def test_model(epoch,Glow_net,train_loader,device,optimizer,scheduler):\r\n",
        "  #global test_loss\r\n",
        "  Glow_net.eval()\r\n",
        "  loss_meter=AverageMeter()\r\n",
        "  with tqdm(total=len(test_loader.dataset)) as progress_bar:\r\n",
        "        for image,label in test_loader:\r\n",
        "            x=image.to(device)\r\n",
        "            z,det=Glow_net.inference(x)\r\n",
        "            prior= -0.5 * (z**2 + np.log(2*np.pi))\r\n",
        "            #log_ll=torch.sum(Glow_net.prior.log_prob(z),dim=[1,2,3])\r\n",
        "            prior_ll=prior.flatten(1).sum(-1) - np.log(256) * np.prod(z.size()[1:])\r\n",
        "            #log_ll=log_ll-np.log(256)*np.prod(z.size()[1:])\r\n",
        "            loss=-(prior_ll+det).mean()\r\n",
        "            loss_meter.update(loss.item(),x.size(0))\r\n",
        "            progress_bar.set_postfix(nll=loss_meter.avg,bpd=bits_per_dim(x,loss_meter.avg),)\r\n",
        "            progress_bar.update(x.size(0))\r\n",
        "  images=Glow_net.sample_images(64,1,32,32)\r\n",
        "  images1=torch.sigmoid(images)\r\n",
        "  #os.makedirs('samples',exist_ok=True)\r\n",
        "  images_concat=torchvision.utils.make_grid(images,nrow=int(64 ** 0.5), padding=2, pad_value=255)\r\n",
        "  torchvision.utils.save_image(images_concat,'new_samples/epoch_{}.png'.format(epoch))\r\n",
        "  images_concat1=torchvision.utils.make_grid(images1,nrow=int(64 ** 0.5), padding=2, pad_value=255)\r\n",
        "  torchvision.utils.save_image(images_concat1,'new_samples/epoch1_{}.png'.format(epoch))\r\n",
        "  #return (images)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dfMjE2vDywaS"
      },
      "source": [
        "## BEFORE RESTARTING TRAINING ADD MEAN.PT FILE AND RECHECK GENARATED RESULTS"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5_J8RymoYcj3",
        "outputId": "2dc51d3e-e32d-427c-f8b6-54333f755d23"
      },
      "source": [
        "global_step=0\r\n",
        "start_epoch=0\r\n",
        "num_epochs=100\r\n",
        "for epoch in range(start_epoch, start_epoch + num_epochs):\r\n",
        "  train_model(epoch,Glow_net,train_loader,device,optimizer,scheduler)\r\n",
        "  test_model(epoch,Glow_net,test_loader,device,optimizer,scheduler)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:35<00:00, 131.77it/s, bpd=4.8, lr=1.88e-6, nll=3.41e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.06it/s, bpd=3.74, nll=2.66e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:35<00:00, 131.64it/s, bpd=3.55, lr=3.75e-6, nll=2.52e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.91it/s, bpd=3.4, nll=2.41e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:34<00:00, 132.02it/s, bpd=3.34, lr=5.63e-6, nll=2.37e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 334.05it/s, bpd=3.27, nll=2.32e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:35<00:00, 131.83it/s, bpd=3.24, lr=7.5e-6, nll=2.3e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 335.36it/s, bpd=3.18, nll=2.26e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:34<00:00, 132.05it/s, bpd=3.17, lr=9.38e-6, nll=2.25e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 336.21it/s, bpd=3.19, nll=2.26e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:34<00:00, 132.03it/s, bpd=3.12, lr=1.13e-5, nll=2.21e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 334.74it/s, bpd=3.1, nll=2.2e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.51it/s, bpd=3.09, lr=1.31e-5, nll=2.19e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.82it/s, bpd=3.06, nll=2.17e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.42it/s, bpd=3.05, lr=1.5e-5, nll=2.17e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.90it/s, bpd=3.04, nll=2.16e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.54it/s, bpd=3.03, lr=1.69e-5, nll=2.15e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.05it/s, bpd=3.02, nll=2.15e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.31it/s, bpd=3.01, lr=1.88e-5, nll=2.14e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 334.70it/s, bpd=3.06, nll=2.17e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.47it/s, bpd=2.99, lr=2.06e-5, nll=2.12e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.04it/s, bpd=3.05, nll=2.16e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.74it/s, bpd=2.98, lr=2.25e-5, nll=2.11e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.48it/s, bpd=3, nll=2.13e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:40<00:00, 130.42it/s, bpd=2.97, lr=2.44e-5, nll=2.11e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 328.87it/s, bpd=2.96, nll=2.1e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:40<00:00, 130.28it/s, bpd=2.95, lr=2.63e-5, nll=2.1e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 330.14it/s, bpd=2.97, nll=2.11e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:40<00:00, 130.29it/s, bpd=2.94, lr=2.81e-5, nll=2.09e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 330.21it/s, bpd=2.99, nll=2.12e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.78it/s, bpd=2.93, lr=3e-5, nll=2.08e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 330.05it/s, bpd=3.02, nll=2.14e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.88it/s, bpd=2.92, lr=3.19e-5, nll=2.07e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.29it/s, bpd=3.01, nll=2.14e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.33it/s, bpd=2.91, lr=3.38e-5, nll=2.07e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.65it/s, bpd=2.95, nll=2.09e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:39<00:00, 130.59it/s, bpd=2.91, lr=3.56e-5, nll=2.06e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 328.62it/s, bpd=2.95, nll=2.09e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.08it/s, bpd=2.89, lr=3.75e-5, nll=2.05e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 333.17it/s, bpd=3.02, nll=2.14e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.20it/s, bpd=2.89, lr=3.94e-5, nll=2.05e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.00it/s, bpd=2.92, nll=2.07e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.89it/s, bpd=2.88, lr=4.13e-5, nll=2.05e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.68it/s, bpd=2.86, nll=2.03e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.07it/s, bpd=2.88, lr=4.31e-5, nll=2.04e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.86it/s, bpd=2.93, nll=2.08e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:39<00:00, 130.49it/s, bpd=2.87, lr=4.5e-5, nll=2.04e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 329.01it/s, bpd=2.95, nll=2.09e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:40<00:00, 130.17it/s, bpd=2.86, lr=4.69e-5, nll=2.03e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 327.98it/s, bpd=2.95, nll=2.09e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:41<00:00, 129.97it/s, bpd=2.86, lr=4.88e-5, nll=2.03e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 329.14it/s, bpd=2.9, nll=2.05e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.96it/s, bpd=2.85, lr=5.07e-5, nll=2.02e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 330.79it/s, bpd=2.93, nll=2.08e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:39<00:00, 130.45it/s, bpd=2.85, lr=5.25e-5, nll=2.02e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 328.43it/s, bpd=2.89, nll=2.05e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:41<00:00, 130.12it/s, bpd=2.84, lr=5.44e-5, nll=2.02e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 327.05it/s, bpd=2.93, nll=2.08e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:39<00:00, 130.55it/s, bpd=2.83, lr=5.63e-5, nll=2.01e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.04it/s, bpd=2.96, nll=2.1e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.27it/s, bpd=2.83, lr=5.82e-5, nll=2.01e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 333.16it/s, bpd=3.03, nll=2.15e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 31\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.91it/s, bpd=2.82, lr=6e-5, nll=2e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.16it/s, bpd=2.95, nll=2.09e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 32\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.13it/s, bpd=2.82, lr=6.19e-5, nll=2e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 333.92it/s, bpd=2.89, nll=2.05e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 33\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.08it/s, bpd=2.82, lr=6.38e-5, nll=2e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.54it/s, bpd=2.88, nll=2.05e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 34\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:39<00:00, 130.61it/s, bpd=2.81, lr=6.57e-5, nll=1.99e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.22it/s, bpd=2.89, nll=2.05e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 35\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.94it/s, bpd=2.81, lr=6.75e-5, nll=1.99e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 330.62it/s, bpd=2.83, nll=2.01e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 36\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:38<00:00, 130.89it/s, bpd=2.81, lr=6.94e-5, nll=1.99e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.35it/s, bpd=2.84, nll=2.02e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 37\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:39<00:00, 130.48it/s, bpd=2.8, lr=7.13e-5, nll=1.99e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.85it/s, bpd=2.85, nll=2.02e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 38\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.38it/s, bpd=2.8, lr=7.32e-5, nll=1.98e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 333.35it/s, bpd=2.82, nll=2e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 39\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:37<00:00, 131.07it/s, bpd=2.79, lr=7.5e-5, nll=1.98e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 331.57it/s, bpd=2.92, nll=2.07e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 40\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.40it/s, bpd=2.79, lr=7.69e-5, nll=1.98e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.27it/s, bpd=2.83, nll=2.01e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 41\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.56it/s, bpd=2.78, lr=7.88e-5, nll=1.97e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 333.72it/s, bpd=2.81, nll=1.99e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 42\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:36<00:00, 131.33it/s, bpd=2.78, lr=8.07e-5, nll=1.97e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 333.90it/s, bpd=2.93, nll=2.08e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 43\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:33<00:00, 132.36it/s, bpd=2.77, lr=8.25e-5, nll=1.97e+3]\n",
            "100%|██████████| 10000/10000 [00:30<00:00, 332.63it/s, bpd=2.83, nll=2.01e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 44\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:33<00:00, 132.30it/s, bpd=2.77, lr=8.44e-5, nll=1.97e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 339.75it/s, bpd=2.95, nll=2.1e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 45\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:30<00:00, 133.33it/s, bpd=2.76, lr=8.63e-5, nll=1.96e+3]\n",
            "100%|██████████| 10000/10000 [00:29<00:00, 337.97it/s, bpd=2.88, nll=2.04e+3]\n",
            "  0%|          | 0/60000 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 46\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 60000/60000 [07:29<00:00, 133.58it/s, bpd=2.76, lr=8.82e-5, nll=1.96e+3]\n",
            " 54%|█████▍    | 5376/10000 [00:15<00:13, 345.79it/s, bpd=2.77, nll=1.97e+3]"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ysUOMd70JO5m"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}