Int_div_mod3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "10ddc7c4180e4e09a619bba9df87a291": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33ff2f94716b4e0286a7999ae881414a",
              "IPY_MODEL_231bbea44ffe41cb8c523be1e159e48e",
              "IPY_MODEL_37f14f830cdf456cb64598f9c2e97975"
            ],
            "layout": "IPY_MODEL_ac303bced2594a66bf187bed539382ae"
          }
        },
        "33ff2f94716b4e0286a7999ae881414a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0249b9e428cd48e1a5684d539a2c79af",
            "placeholder": "​",
            "style": "IPY_MODEL_93a6691533a7450097593a0901d93a3c",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "231bbea44ffe41cb8c523be1e159e48e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc9fb1754ca44fa18df7da3c9d6819fb",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e5bb5b32952c47afa4b7e8aef3e81358",
            "value": 570
          }
        },
        "37f14f830cdf456cb64598f9c2e97975": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f72fb7926214eefbc26c74b7c703cc0",
            "placeholder": "​",
            "style": "IPY_MODEL_b123ceecc0df4c3995845dacbc8f143f",
            "value": " 570/570 [00:00&lt;00:00, 14.3kB/s]"
          }
        },
        "ac303bced2594a66bf187bed539382ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0249b9e428cd48e1a5684d539a2c79af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93a6691533a7450097593a0901d93a3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dc9fb1754ca44fa18df7da3c9d6819fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5bb5b32952c47afa4b7e8aef3e81358": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6f72fb7926214eefbc26c74b7c703cc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b123ceecc0df4c3995845dacbc8f143f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc1c384765ee44d2b4ca9cf97d7c84d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a8ffdd5d5c740f3a14c31d65835b677",
              "IPY_MODEL_55e20724446342b0a9e49425587eff67",
              "IPY_MODEL_862ed75832214a48b6cd9e4bd476757a"
            ],
            "layout": "IPY_MODEL_7fecb74fbf0a4f3588a20a30c1c9f838"
          }
        },
        "2a8ffdd5d5c740f3a14c31d65835b677": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_79e78afe7a6a49d8bc2a1339cb608986",
            "placeholder": "​",
            "style": "IPY_MODEL_408816b23ad6497a9d058c40d1e4447a",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "55e20724446342b0a9e49425587eff67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca60f14371184bdda15a5ae11dd2e4d5",
            "max": 435755784,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_68dff87eeb80427d8339a7051e140c44",
            "value": 435755784
          }
        },
        "862ed75832214a48b6cd9e4bd476757a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e20e4a43beb494bab4c29a4c063bdd0",
            "placeholder": "​",
            "style": "IPY_MODEL_665d384791f242d0a4c2698399e8c476",
            "value": " 436M/436M [00:05&lt;00:00, 103MB/s]"
          }
        },
        "7fecb74fbf0a4f3588a20a30c1c9f838": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79e78afe7a6a49d8bc2a1339cb608986": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "408816b23ad6497a9d058c40d1e4447a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca60f14371184bdda15a5ae11dd2e4d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68dff87eeb80427d8339a7051e140c44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5e20e4a43beb494bab4c29a4c063bdd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "665d384791f242d0a4c2698399e8c476": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4e75f51cdb114436bb6077039a2bceac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3413d7123c2c455e946d245d9c38a1d9",
              "IPY_MODEL_cbf937c3ef7a4ff7b3c65264ceec2b02",
              "IPY_MODEL_0c5997f4f56e4dffaecbca1a59aaf1ed"
            ],
            "layout": "IPY_MODEL_22854886768044f3938ef35f36eb4c56"
          }
        },
        "3413d7123c2c455e946d245d9c38a1d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a88f442f525402f9898ce2638f060e3",
            "placeholder": "​",
            "style": "IPY_MODEL_3d6d989538a240aa891c8fe33d8233b8",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "cbf937c3ef7a4ff7b3c65264ceec2b02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1a78a63da36746ecb4e7776e76ec5843",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1324d7d26c414135a0e8933dc04cde1e",
            "value": 570
          }
        },
        "0c5997f4f56e4dffaecbca1a59aaf1ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2074af926914628998697cf51a5d5a3",
            "placeholder": "​",
            "style": "IPY_MODEL_bb95321703f94413b31f1d0fae9f755f",
            "value": " 570/570 [00:00&lt;00:00, 21.3kB/s]"
          }
        },
        "22854886768044f3938ef35f36eb4c56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a88f442f525402f9898ce2638f060e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d6d989538a240aa891c8fe33d8233b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1a78a63da36746ecb4e7776e76ec5843": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1324d7d26c414135a0e8933dc04cde1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c2074af926914628998697cf51a5d5a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb95321703f94413b31f1d0fae9f755f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eaeee2077bef446ea0d01ffd139c6bef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a108833940174e9e87b27d0613a7201b",
              "IPY_MODEL_cf367f3b85dd43f3b549f6c7a90489d8",
              "IPY_MODEL_81314df1ccbb47ecbd68de5a1bdf75ff"
            ],
            "layout": "IPY_MODEL_d1eda5beb978466ca95f72bfb9039594"
          }
        },
        "a108833940174e9e87b27d0613a7201b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67b3fb55f97a481fbf11aef380326d53",
            "placeholder": "​",
            "style": "IPY_MODEL_88962dad58e44a36b63e2ec9e05ec6f4",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "cf367f3b85dd43f3b549f6c7a90489d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7285d9af6b044ecebf6196185930afbb",
            "max": 435755784,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8346bd48676e4ea18432ab4e95e0fd83",
            "value": 435755784
          }
        },
        "81314df1ccbb47ecbd68de5a1bdf75ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1edcc63ae94d4557985d861b2c7a121a",
            "placeholder": "​",
            "style": "IPY_MODEL_64a50ea0b74f4faab9ab4a4e92047200",
            "value": " 436M/436M [00:02&lt;00:00, 218MB/s]"
          }
        },
        "d1eda5beb978466ca95f72bfb9039594": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "67b3fb55f97a481fbf11aef380326d53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88962dad58e44a36b63e2ec9e05ec6f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7285d9af6b044ecebf6196185930afbb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8346bd48676e4ea18432ab4e95e0fd83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1edcc63ae94d4557985d861b2c7a121a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64a50ea0b74f4faab9ab4a4e92047200": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "_pCjPw1gfWpM"
      },
      "outputs": [],
      "source": [
        "#import packages\n",
        "import numpy as np\n",
        "import random\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, LSTM\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import sys\n",
        "# set up parameters\n",
        "NUM_DIGITS = 32\n",
        "NUM_TRAIN_EXAMPLES = 50000\n",
        "EPOCHS = 10\n",
        "BATCH_SIZE = 32"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - no feature engineering\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "# Convert to tensor form\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "#Define model\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(1,)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "#Compile model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "#Train model\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nfaM9aiyhaiZ",
        "outputId": "850715ec-8e96-4c4a-e54b-47bccca4b9df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 12s 4ms/step - loss: 6603657.0000 - accuracy: 0.3314 - val_loss: 2369414.0000 - val_accuracy: 0.3266\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 5255542.0000 - accuracy: 0.3335 - val_loss: 992871.7500 - val_accuracy: 0.3312\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 4619045.5000 - accuracy: 0.3290 - val_loss: 4875983.5000 - val_accuracy: 0.3312\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 3487551.7500 - accuracy: 0.3332 - val_loss: 4116714.7500 - val_accuracy: 0.3312\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 3016698.0000 - accuracy: 0.3320 - val_loss: 790843.0000 - val_accuracy: 0.3312\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 2543549.5000 - accuracy: 0.3362 - val_loss: 751854.5000 - val_accuracy: 0.3266\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1845404.0000 - accuracy: 0.3373 - val_loss: 1258259.0000 - val_accuracy: 0.3422\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1226673.7500 - accuracy: 0.3303 - val_loss: 485355.7188 - val_accuracy: 0.3266\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 937834.2500 - accuracy: 0.3358 - val_loss: 407393.7500 - val_accuracy: 0.3266\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 553161.3125 - accuracy: 0.3374 - val_loss: 1.1000 - val_accuracy: 0.3312\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba887ffb50>"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - binary representation\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "#Convert to binary representation\n",
        "def to_binary(x):\n",
        "    return [int(d) for d in '{0:b}'.format(x).zfill(NUM_DIGITS)]\n",
        "\n",
        "X_train = [to_binary(x) for x in X_train]\n",
        "\n",
        "\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(NUM_DIGITS,)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "#Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kKqoNeAJhkks",
        "outputId": "b30c0654-bff7-4a04-b17e-d392b93b06d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 7s 4ms/step - loss: 1.1010 - accuracy: 0.3398 - val_loss: 1.0998 - val_accuracy: 0.3292\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0985 - accuracy: 0.3398 - val_loss: 1.0990 - val_accuracy: 0.3314\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0981 - accuracy: 0.3449 - val_loss: 1.1001 - val_accuracy: 0.3412\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0970 - accuracy: 0.3522 - val_loss: 1.1009 - val_accuracy: 0.3326\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0956 - accuracy: 0.3567 - val_loss: 1.1003 - val_accuracy: 0.3378\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0941 - accuracy: 0.3637 - val_loss: 1.1018 - val_accuracy: 0.3276\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0919 - accuracy: 0.3692 - val_loss: 1.1015 - val_accuracy: 0.3442\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0892 - accuracy: 0.3761 - val_loss: 1.1016 - val_accuracy: 0.3434\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0864 - accuracy: 0.3847 - val_loss: 1.1063 - val_accuracy: 0.3330\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0832 - accuracy: 0.3889 - val_loss: 1.1067 - val_accuracy: 0.3454\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba8870fe50>"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - base three representation\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "def int_to_base3(num):\n",
        "    # Convert to base-3 list of integers\n",
        "    base3_list = []\n",
        "    while num > 0:\n",
        "        base3_list.insert(0, num % 3)\n",
        "        num //= 3\n",
        "\n",
        "    # Pad with zeros up to 32 digits\n",
        "    while len(base3_list) < 32:\n",
        "        base3_list.insert(0, 0)\n",
        "\n",
        "    return base3_list\n",
        "\n",
        "X_train = [int_to_base3(x) for x in X_train]\n",
        "\n",
        "# Convert to tensor form\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "# Define model\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(32,)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "# Compile model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "# Train model\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TE9rGcG6iPV0",
        "outputId": "b7c6c4bc-f2c3-49d7-8276-94076450e80a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 7s 4ms/step - loss: 0.0990 - accuracy: 0.9686 - val_loss: 9.3387e-04 - val_accuracy: 1.0000\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 3.8559e-04 - accuracy: 1.0000 - val_loss: 1.4516e-04 - val_accuracy: 1.0000\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 7.9654e-05 - accuracy: 1.0000 - val_loss: 4.5313e-05 - val_accuracy: 1.0000\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 2.5797e-05 - accuracy: 1.0000 - val_loss: 1.6559e-05 - val_accuracy: 1.0000\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 9.9448e-06 - accuracy: 1.0000 - val_loss: 6.6919e-06 - val_accuracy: 1.0000\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 4.0482e-06 - accuracy: 1.0000 - val_loss: 2.8641e-06 - val_accuracy: 1.0000\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.7237e-06 - accuracy: 1.0000 - val_loss: 1.2390e-06 - val_accuracy: 1.0000\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 7.4096e-07 - accuracy: 1.0000 - val_loss: 5.7779e-07 - val_accuracy: 1.0000\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 3.2123e-07 - accuracy: 1.0000 - val_loss: 2.5127e-07 - val_accuracy: 1.0000\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.3780e-07 - accuracy: 1.0000 - val_loss: 1.0176e-07 - val_accuracy: 1.0000\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb9e88cdf30>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - one gram\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x) for x in X_train]\n",
        "\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = [[0]*(NUM_DIGITS-len(x))+x for x in X_train]\n",
        "\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(NUM_DIGITS,)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zmxVxpG7jrjA",
        "outputId": "0cbe310c-4d27-41d3-99ee-ec3416df22a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 7s 4ms/step - loss: 1.1106 - accuracy: 0.3344 - val_loss: 1.1062 - val_accuracy: 0.3336\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1.1010 - accuracy: 0.3413 - val_loss: 1.1010 - val_accuracy: 0.3454\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0998 - accuracy: 0.3380 - val_loss: 1.1007 - val_accuracy: 0.3390\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0988 - accuracy: 0.3410 - val_loss: 1.0987 - val_accuracy: 0.3384\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0983 - accuracy: 0.3481 - val_loss: 1.1000 - val_accuracy: 0.3366\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0980 - accuracy: 0.3412 - val_loss: 1.1004 - val_accuracy: 0.3214\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0979 - accuracy: 0.3448 - val_loss: 1.1005 - val_accuracy: 0.3330\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0973 - accuracy: 0.3495 - val_loss: 1.1000 - val_accuracy: 0.3304\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0969 - accuracy: 0.3487 - val_loss: 1.1011 - val_accuracy: 0.3280\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0965 - accuracy: 0.3540 - val_loss: 1.0997 - val_accuracy: 0.3316\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba059c8850>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - one gram + its sum\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x)+[sum(to_sequence(x))] for x in X_train]\n",
        "\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = [[0]*(NUM_DIGITS-len(x))+x for x in X_train]\n",
        "\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(NUM_DIGITS,)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pPxsqmA9jJff",
        "outputId": "4bc1fcf6-5caf-41a3-c8db-333b2944f34b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 8s 4ms/step - loss: 1.1408 - accuracy: 0.3312 - val_loss: 1.1167 - val_accuracy: 0.3400\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.1133 - accuracy: 0.3333 - val_loss: 1.1053 - val_accuracy: 0.3378\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.1094 - accuracy: 0.3320 - val_loss: 1.1043 - val_accuracy: 0.3268\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.1049 - accuracy: 0.3333 - val_loss: 1.1052 - val_accuracy: 0.3370\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.1020 - accuracy: 0.3344 - val_loss: 1.1039 - val_accuracy: 0.3346\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.1003 - accuracy: 0.3362 - val_loss: 1.0994 - val_accuracy: 0.3338\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1.0988 - accuracy: 0.3376 - val_loss: 1.0989 - val_accuracy: 0.3342\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0988 - accuracy: 0.3356 - val_loss: 1.0989 - val_accuracy: 0.3326\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0986 - accuracy: 0.3393 - val_loss: 1.0992 - val_accuracy: 0.3302\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 1.0986 - accuracy: 0.3318 - val_loss: 1.0987 - val_accuracy: 0.3250\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba057368f0>"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - one gram + (its sum % 3)\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x)+[sum(to_sequence(x)) % 3] for x in X_train]\n",
        "\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = [[0]*(NUM_DIGITS-len(x))+x for x in X_train]\n",
        "\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(NUM_DIGITS,)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6H0RMircjhpa",
        "outputId": "b09eddce-ca32-4d56-fffc-41da9275fffb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 0.1461 - accuracy: 0.9495 - val_loss: 0.0025 - val_accuracy: 1.0000\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 0.0013 - accuracy: 1.0000 - val_loss: 3.8451e-04 - val_accuracy: 1.0000\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 2.6021e-04 - accuracy: 1.0000 - val_loss: 1.1125e-04 - val_accuracy: 1.0000\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 8.4427e-05 - accuracy: 1.0000 - val_loss: 4.0761e-05 - val_accuracy: 1.0000\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 3.1645e-05 - accuracy: 1.0000 - val_loss: 1.6422e-05 - val_accuracy: 1.0000\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.1294e-05 - accuracy: 1.0000 - val_loss: 6.2047e-06 - val_accuracy: 1.0000\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 4.4605e-06 - accuracy: 1.0000 - val_loss: 2.4842e-06 - val_accuracy: 1.0000\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.8300e-06 - accuracy: 1.0000 - val_loss: 1.1504e-06 - val_accuracy: 1.0000\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 5s 4ms/step - loss: 8.0303e-07 - accuracy: 1.0000 - val_loss: 4.9743e-07 - val_accuracy: 1.0000\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 3.4997e-07 - accuracy: 1.0000 - val_loss: 2.2106e-07 - val_accuracy: 1.0000\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba042d85b0>"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - two gram\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "# two_gram_consec encoding\n",
        "def two_gram_consec(num):\n",
        "\n",
        "    str_num = str(num).zfill(32)\n",
        "\n",
        "    two_grams = []\n",
        "\n",
        "    for i in range(31):\n",
        "        two_gram = [int(str_num[i]), int(str_num[i+1])]\n",
        "        two_grams.append(two_gram)\n",
        "\n",
        "    return two_grams\n",
        "\n",
        "X_train = [two_gram_consec(x) for x in X_train]\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "\n",
        "# Convert to tensor form\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "# Define model\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(31,2)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Flatten(),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "# Compile model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "# Train model\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6-EBcaFZj9kx",
        "outputId": "ed94ef9e-2961-4abf-fa99-3d7f7d6f7723"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 9s 5ms/step - loss: 1.1002 - accuracy: 0.3332 - val_loss: 1.0992 - val_accuracy: 0.3066\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0987 - accuracy: 0.3333 - val_loss: 1.0988 - val_accuracy: 0.3066\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3339 - val_loss: 1.0991 - val_accuracy: 0.3406\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 6s 4ms/step - loss: 1.0987 - accuracy: 0.3317 - val_loss: 1.0989 - val_accuracy: 0.3066\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3308 - val_loss: 1.0989 - val_accuracy: 0.3406\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1.0987 - accuracy: 0.3312 - val_loss: 1.0988 - val_accuracy: 0.3066\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0986 - accuracy: 0.3346 - val_loss: 1.0989 - val_accuracy: 0.3406\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3299 - val_loss: 1.0989 - val_accuracy: 0.3406\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0987 - accuracy: 0.3330 - val_loss: 1.0986 - val_accuracy: 0.3406\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3316 - val_loss: 1.0996 - val_accuracy: 0.3066\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb9c079a2f0>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - three gram encoding\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "# three_gram_consec encoding\n",
        "def three_gram_consec(num):\n",
        "    # Convert the integer to a string with 32+1 digits\n",
        "    str_num = str(num).zfill(32)\n",
        "\n",
        "    # Create a list to store the three-grams\n",
        "    three_grams = []\n",
        "\n",
        "    # Iterate over the string and create three-grams\n",
        "    for i in range(30):\n",
        "        three_gram = [int(str_num[i]), int(str_num[i+1]), int(str_num[i+2])]\n",
        "        three_grams.append(three_gram)\n",
        "\n",
        "    # Return the list of three-grams\n",
        "    return three_grams\n",
        "\n",
        "X_train = [three_gram_consec(x) for x in X_train]\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "\n",
        "# Convert to tensor form\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "# Define model\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(30,3)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Flatten(),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "# Compile model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "# Train model\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xXtPPfMzkSZz",
        "outputId": "6bfcd399-2fd1-4043-8149-f351cdfc3dc9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 9s 5ms/step - loss: 1.1007 - accuracy: 0.3334 - val_loss: 1.0987 - val_accuracy: 0.3308\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0988 - accuracy: 0.3295 - val_loss: 1.0986 - val_accuracy: 0.3408\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3360 - val_loss: 1.0986 - val_accuracy: 0.3348\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3368 - val_loss: 1.0985 - val_accuracy: 0.3382\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3332 - val_loss: 1.0987 - val_accuracy: 0.3288\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3315 - val_loss: 1.0987 - val_accuracy: 0.3288\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3339 - val_loss: 1.0987 - val_accuracy: 0.3330\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3288 - val_loss: 1.0987 - val_accuracy: 0.3330\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3326 - val_loss: 1.0988 - val_accuracy: 0.3288\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3288 - val_loss: 1.0986 - val_accuracy: 0.3330\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb9c0799270>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN -one + two gram\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "def one_gram_plus_two_gram(num):\n",
        "    num_str = str(num).zfill(32)\n",
        "    one_gram = [[0,0,int(x)] for x in num_str]\n",
        "    two_gram = [[0,int(num_str[i]), int(num_str[i+1])] for i in range(31)]\n",
        "    return one_gram+two_gram\n",
        "\n",
        "X_train = [one_gram_plus_two_gram(x) for x in X_train]\n",
        "\n",
        "\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(63,3)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Flatten(),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8wzpni26kphB",
        "outputId": "6494e822-0eb1-4531-99f5-3e77df7d9e7b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 9s 6ms/step - loss: 1.1013 - accuracy: 0.3322 - val_loss: 1.0989 - val_accuracy: 0.3280\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0986 - accuracy: 0.3362 - val_loss: 1.0984 - val_accuracy: 0.3444\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3331 - val_loss: 1.0988 - val_accuracy: 0.3280\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3362 - val_loss: 1.0991 - val_accuracy: 0.3280\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0987 - accuracy: 0.3338 - val_loss: 1.0988 - val_accuracy: 0.3276\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3314 - val_loss: 1.0991 - val_accuracy: 0.3276\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0986 - accuracy: 0.3349 - val_loss: 1.0990 - val_accuracy: 0.3276\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3303 - val_loss: 1.0989 - val_accuracy: 0.3276\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3323 - val_loss: 1.0988 - val_accuracy: 0.3276\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0986 - accuracy: 0.3357 - val_loss: 1.0985 - val_accuracy: 0.3444\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb984cd3e20>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#ANN - one + two + three gram\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = [[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train]\n",
        "\n",
        "def multi_gram(num):\n",
        "    num_str = str(num).zfill(32)\n",
        "    one_gram = [[0,0,int(x)] for x in num_str]\n",
        "    two_gram = [[0,int(num_str[i]), int(num_str[i+1])] for i in range(31)]\n",
        "    three_gram = [[int(num_str[i]), int(num_str[i+1]), int(num_str[i+2])] for i in range(30)]\n",
        "    return one_gram+two_gram+three_gram\n",
        "\n",
        "X_train = [multi_gram(x) for x in X_train]\n",
        "\n",
        "\n",
        "X_train = tf.convert_to_tensor(X_train)\n",
        "y_train = tf.convert_to_tensor(y_train)\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Dense(64, activation='relu', input_shape=(93,3)),\n",
        "    tf.keras.layers.Dense(32, activation='relu'),\n",
        "    tf.keras.layers.Flatten(),\n",
        "    tf.keras.layers.Dense(3, activation='softmax')\n",
        "])\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X_train, y_train, validation_split = 0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j2AUr-OkkwTX",
        "outputId": "fc0c00e3-8e6f-4cb3-da3e-9d0abfb9b695"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 9s 5ms/step - loss: 1.1018 - accuracy: 0.3368 - val_loss: 1.0986 - val_accuracy: 0.3334\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3302 - val_loss: 1.0987 - val_accuracy: 0.3352\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1.0987 - accuracy: 0.3333 - val_loss: 1.0986 - val_accuracy: 0.3282\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3325 - val_loss: 1.0986 - val_accuracy: 0.3282\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3314 - val_loss: 1.0986 - val_accuracy: 0.3326\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0987 - accuracy: 0.3320 - val_loss: 1.0987 - val_accuracy: 0.3326\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3322 - val_loss: 1.0986 - val_accuracy: 0.3326\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3359 - val_loss: 1.0987 - val_accuracy: 0.3282\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3304 - val_loss: 1.0986 - val_accuracy: 0.3326\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3338 - val_loss: 1.0987 - val_accuracy: 0.3282\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb979650730>"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#CNN - one gram\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = np.array([[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train])\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x) for x in X_train]\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = np.array([[0]*(NUM_DIGITS-len(x))+x for x in X_train])\n",
        "\n",
        "# Reshape the data for CNN input\n",
        "X_train = X_train.reshape(-1, NUM_DIGITS, 1)\n",
        "\n",
        "# Define the model\n",
        "model = Sequential()\n",
        "model.add(Conv1D(32, kernel_size=3, activation='relu', input_shape=(NUM_DIGITS, 1)))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Conv1D(64, kernel_size=3, activation='relu'))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dense(3, activation='sigmoid'))\n",
        "\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "# Train the model\n",
        "model.fit(X_train, y_train, validation_split=0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VZfWN70LmiVj",
        "outputId": "2266863b-6d55-44ea-e50e-90368f05afec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 13s 5ms/step - loss: 1.1007 - accuracy: 0.3323 - val_loss: 1.0990 - val_accuracy: 0.3346\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0989 - accuracy: 0.3356 - val_loss: 1.0989 - val_accuracy: 0.3364\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3388 - val_loss: 1.0987 - val_accuracy: 0.3334\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3328 - val_loss: 1.0986 - val_accuracy: 0.3344\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3356 - val_loss: 1.0991 - val_accuracy: 0.3276\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3354 - val_loss: 1.0989 - val_accuracy: 0.3306\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3334 - val_loss: 1.0987 - val_accuracy: 0.3314\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0986 - accuracy: 0.3358 - val_loss: 1.0987 - val_accuracy: 0.3308\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0986 - accuracy: 0.3334 - val_loss: 1.0987 - val_accuracy: 0.3340\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0985 - accuracy: 0.3379 - val_loss: 1.0993 - val_accuracy: 0.3286\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fba054ebcd0>"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#CNN - one gram + its sum\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = np.array([[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train])\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x)+[sum(to_sequence(x))] for x in X_train]\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = np.array([[0]*(NUM_DIGITS-len(x))+x for x in X_train])\n",
        "\n",
        "# Reshape the data for CNN input\n",
        "X_train = X_train.reshape(-1, NUM_DIGITS, 1)\n",
        "\n",
        "# Define the model\n",
        "model = Sequential()\n",
        "model.add(Conv1D(32, kernel_size=3, activation='relu', input_shape=(NUM_DIGITS, 1)))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Conv1D(64, kernel_size=3, activation='relu'))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dense(3, activation='sigmoid'))\n",
        "\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "# Train the model\n",
        "model.fit(X_train, y_train, validation_split=0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UNqGokHmlJMH",
        "outputId": "c4a6ada1-684e-4951-9d38-8873a7fa08dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 9s 5ms/step - loss: 1.1013 - accuracy: 0.3328 - val_loss: 1.0987 - val_accuracy: 0.3386\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0990 - accuracy: 0.3356 - val_loss: 1.0986 - val_accuracy: 0.3402\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0989 - accuracy: 0.3328 - val_loss: 1.0985 - val_accuracy: 0.3356\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3342 - val_loss: 1.0986 - val_accuracy: 0.3380\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3354 - val_loss: 1.0992 - val_accuracy: 0.3274\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0987 - accuracy: 0.3368 - val_loss: 1.0988 - val_accuracy: 0.3288\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0986 - accuracy: 0.3353 - val_loss: 1.0988 - val_accuracy: 0.3336\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0986 - accuracy: 0.3369 - val_loss: 1.0987 - val_accuracy: 0.3340\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0985 - accuracy: 0.3408 - val_loss: 1.0997 - val_accuracy: 0.3304\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0985 - accuracy: 0.3404 - val_loss: 1.0986 - val_accuracy: 0.3370\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb96f0f2920>"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#CNN - one gram + (its sum % 3)\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = np.array([[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train])\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x)+[sum(to_sequence(x)) % 3] for x in X_train]\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = np.array([[0]*(NUM_DIGITS-len(x))+x for x in X_train])\n",
        "\n",
        "# Reshape the data for CNN input\n",
        "X_train = X_train.reshape(-1, NUM_DIGITS, 1)\n",
        "\n",
        "# Define the model\n",
        "model = Sequential()\n",
        "model.add(Conv1D(32, kernel_size=3, activation='relu', input_shape=(NUM_DIGITS, 1)))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Conv1D(64, kernel_size=3, activation='relu'))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(64, activation='relu'))\n",
        "model.add(Dense(3, activation='sigmoid'))\n",
        "\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "# Train the model\n",
        "model.fit(X_train, y_train, validation_split=0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IyLA956LmxdC",
        "outputId": "fa6e44f3-e46b-4941-ba63-868a02694885"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 10s 5ms/step - loss: 1.1008 - accuracy: 0.3306 - val_loss: 1.0992 - val_accuracy: 0.3278\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0988 - accuracy: 0.3372 - val_loss: 1.0992 - val_accuracy: 0.3308\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1.0986 - accuracy: 0.3356 - val_loss: 1.0985 - val_accuracy: 0.3448\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0986 - accuracy: 0.3417 - val_loss: 1.0991 - val_accuracy: 0.3284\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 6s 5ms/step - loss: 1.0987 - accuracy: 0.3345 - val_loss: 1.0985 - val_accuracy: 0.3428\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0987 - accuracy: 0.3354 - val_loss: 1.0984 - val_accuracy: 0.3470\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0986 - accuracy: 0.3366 - val_loss: 1.0992 - val_accuracy: 0.3250\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0984 - accuracy: 0.3401 - val_loss: 1.0987 - val_accuracy: 0.3378\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0983 - accuracy: 0.3411 - val_loss: 1.0990 - val_accuracy: 0.3320\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0980 - accuracy: 0.3427 - val_loss: 1.0993 - val_accuracy: 0.3202\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb96f526290>"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#RNN - one gram\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = np.array([[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train])\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x) for x in X_train]\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = np.array([[0]*(11-len(x))+x for x in X_train])\n",
        "# Reshape the data for RNN input\n",
        "X_train = X_train.reshape(-1,11, 1)\n",
        "\n",
        "# Define the model\n",
        "model = Sequential()\n",
        "model.add(LSTM(32, input_shape=(11, 1)))\n",
        "model.add(Dense(3, activation='sigmoid'))\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "# Train the model\n",
        "model.fit(X_train, y_train, validation_split=0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FmOm3WVJnX0T",
        "outputId": "291cb4a2-c598-4c9f-8a98-66b0c71a16d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 12s 6ms/step - loss: 1.1007 - accuracy: 0.3360 - val_loss: 1.1001 - val_accuracy: 0.3384\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0995 - accuracy: 0.3325 - val_loss: 1.0990 - val_accuracy: 0.3362\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0994 - accuracy: 0.3333 - val_loss: 1.0994 - val_accuracy: 0.3338\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0991 - accuracy: 0.3370 - val_loss: 1.1003 - val_accuracy: 0.3312\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0991 - accuracy: 0.3369 - val_loss: 1.1005 - val_accuracy: 0.3376\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0991 - accuracy: 0.3390 - val_loss: 1.1001 - val_accuracy: 0.3296\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0992 - accuracy: 0.3336 - val_loss: 1.0989 - val_accuracy: 0.3284\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0991 - accuracy: 0.3347 - val_loss: 1.0986 - val_accuracy: 0.3336\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0990 - accuracy: 0.3344 - val_loss: 1.0996 - val_accuracy: 0.3300\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0990 - accuracy: 0.3348 - val_loss: 1.0992 - val_accuracy: 0.3332\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb9796d71f0>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#RNN - one gram + its sum\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = np.array([[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train])\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x)+[sum(to_sequence(x))] for x in X_train]\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = np.array([[0]*(11-len(x))+x for x in X_train])\n",
        "# Reshape the data for RNN input\n",
        "X_train = X_train.reshape(-1,11, 1)\n",
        "\n",
        "# Define the model\n",
        "model = Sequential()\n",
        "model.add(LSTM(32, input_shape=(11, 1)))\n",
        "model.add(Dense(3, activation='sigmoid'))\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "# Train the model\n",
        "model.fit(X_train, y_train, validation_split=0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0aJ9PDe1nHrc",
        "outputId": "fb079f0e-7d6a-48f5-a6fe-c2be477d8f24"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 10s 6ms/step - loss: 1.1005 - accuracy: 0.3330 - val_loss: 1.0994 - val_accuracy: 0.3340\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0999 - accuracy: 0.3331 - val_loss: 1.0987 - val_accuracy: 0.3348\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0997 - accuracy: 0.3335 - val_loss: 1.0993 - val_accuracy: 0.3318\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0996 - accuracy: 0.3345 - val_loss: 1.0993 - val_accuracy: 0.3308\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0995 - accuracy: 0.3319 - val_loss: 1.0988 - val_accuracy: 0.3372\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.0994 - accuracy: 0.3301 - val_loss: 1.0988 - val_accuracy: 0.3356\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0994 - accuracy: 0.3334 - val_loss: 1.1007 - val_accuracy: 0.3330\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0993 - accuracy: 0.3331 - val_loss: 1.0993 - val_accuracy: 0.3370\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 1.0992 - accuracy: 0.3354 - val_loss: 1.0989 - val_accuracy: 0.3252\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.0992 - accuracy: 0.3342 - val_loss: 1.0985 - val_accuracy: 0.3370\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb96f213760>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#RNN - one gram + (its sum % 3)\n",
        "X_train = np.random.randint(low=0, high=2**NUM_DIGITS-1, size=(NUM_TRAIN_EXAMPLES,))\n",
        "y_train = np.array([[1,0,0] if x % 3 == 0 else [0,1,0] if x % 3 == 1 else [0,0,1] for x in X_train])\n",
        "\n",
        "# Convert to sequences of digits\n",
        "def to_sequence(x):\n",
        "    return [int(d) for d in str(x)]\n",
        "\n",
        "X_train = [to_sequence(x)+[sum(to_sequence(x)) % 3] for x in X_train]\n",
        "\n",
        "# Pad the sequences with zeros\n",
        "X_train = np.array([[0]*(11-len(x))+x for x in X_train])\n",
        "# Reshape the data for RNN input\n",
        "X_train = X_train.reshape(-1,11, 1)\n",
        "\n",
        "# Define the model\n",
        "model = Sequential()\n",
        "model.add(LSTM(32, input_shape=(11, 1)))\n",
        "model.add(Dense(3, activation='sigmoid'))\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "# Train the model\n",
        "model.fit(X_train, y_train, validation_split=0.1, epochs=EPOCHS, batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T0CDuyv0nYRC",
        "outputId": "519bca51-753f-4440-9065-a665f4cb406a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "1407/1407 [==============================] - 11s 6ms/step - loss: 0.2578 - accuracy: 0.9505 - val_loss: 0.0224 - val_accuracy: 1.0000\n",
            "Epoch 2/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 0.0097 - accuracy: 1.0000 - val_loss: 0.0040 - val_accuracy: 1.0000\n",
            "Epoch 3/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 0.0023 - accuracy: 1.0000 - val_loss: 0.0013 - val_accuracy: 1.0000\n",
            "Epoch 4/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 8.1465e-04 - accuracy: 1.0000 - val_loss: 5.0146e-04 - val_accuracy: 1.0000\n",
            "Epoch 5/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 3.3198e-04 - accuracy: 1.0000 - val_loss: 2.1412e-04 - val_accuracy: 1.0000\n",
            "Epoch 6/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 1.4450e-04 - accuracy: 1.0000 - val_loss: 9.5279e-05 - val_accuracy: 1.0000\n",
            "Epoch 7/10\n",
            "1407/1407 [==============================] - 7s 5ms/step - loss: 6.5057e-05 - accuracy: 1.0000 - val_loss: 4.3544e-05 - val_accuracy: 1.0000\n",
            "Epoch 8/10\n",
            "1407/1407 [==============================] - 8s 6ms/step - loss: 2.9802e-05 - accuracy: 1.0000 - val_loss: 2.0032e-05 - val_accuracy: 1.0000\n",
            "Epoch 9/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 1.3804e-05 - accuracy: 1.0000 - val_loss: 9.3430e-06 - val_accuracy: 1.0000\n",
            "Epoch 10/10\n",
            "1407/1407 [==============================] - 8s 5ms/step - loss: 6.4659e-06 - accuracy: 1.0000 - val_loss: 4.4181e-06 - val_accuracy: 1.0000\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb9799c76d0>"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#install tranasformers package\n",
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3BXFU-2msbSA",
        "outputId": "d82e3cba-3edf-4961-c8cb-39f41d3ac002"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m62.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m36.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m106.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m86.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import random\n",
        "from transformers import AutoTokenizer, AutoModel, BertConfig, AutoModelForSequenceClassification, BertForSequenceClassification\n",
        "from torch.optim import Adam\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "7w4YN8NMsdD6"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#BERT - one gram\n",
        "def generate_model(num_labels):\n",
        "    model = BertForSequenceClassification.from_pretrained('bert-base-cased', num_labels = num_labels)\n",
        "    config = model.config\n",
        "    model = BertForSequenceClassification(config)\n",
        "    return model\n",
        "\n",
        "class integer_dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, num, prime):\n",
        "        self.input_ids = []\n",
        "        self.labels = []\n",
        "        self.attention_mask = []\n",
        "        for i in range(num):\n",
        "            a = random.randint(0, 2**32)\n",
        "            text = [10]\n",
        "            mask = [1]\n",
        "            count = 0\n",
        "            #sum = 0\n",
        "            self.labels.append(a%prime)\n",
        "            while a != 0:\n",
        "                text.append(a%10)\n",
        "                #sum += a%10\n",
        "                a = a//10\n",
        "                mask.append(1)\n",
        "            #text.append(sum)\n",
        "            #mask.append(1)\n",
        "            for i in range(13-len(text)):\n",
        "                text.append(10)\n",
        "                mask.append(0)\n",
        "            self.input_ids.append(text)\n",
        "            self.attention_mask.append(mask)\n",
        "        self.input_ids = torch.tensor(self.input_ids)\n",
        "        self.labels = torch.tensor(self.labels)\n",
        "        self.attention_mask = torch.tensor(self.attention_mask)\n",
        "\n",
        "    def classes(self):\n",
        "        return self.labels\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {'input_ids':self.input_ids[idx], 'attention_mask' : self.attention_mask[idx], 'labels':self.labels[idx]}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dataset = integer_dataset(50000, 3)\n",
        "dataloader = torch.utils.data.DataLoader(dataset, batch_size=16)\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "model = generate_model(3)\n",
        "\n",
        "\n",
        "def train(model, train_num, val_num, learning_rate, epochs, prime, batch):\n",
        "\n",
        "    train, val = integer_dataset(train_num,prime), integer_dataset(val_num, prime)\n",
        "\n",
        "    train_dataloader = torch.utils.data.DataLoader(train, batch_size=batch, shuffle=True)\n",
        "    val_dataloader = torch.utils.data.DataLoader(val, batch_size=2)\n",
        "\n",
        "\n",
        "    optimizer = Adam(model.parameters(), lr= learning_rate)\n",
        "    model.train()\n",
        "    if use_cuda:\n",
        "\n",
        "            model = model.cuda()\n",
        "\n",
        "    for epoch_num in range(epochs):\n",
        "\n",
        "            total_acc_train = 0\n",
        "            total_loss_train = 0\n",
        "\n",
        "            for batch in tqdm(train_dataloader):\n",
        "\n",
        "                train_label = batch['labels'].to(device)\n",
        "                mask = batch['attention_mask'].to(device)\n",
        "                input_ids = batch['input_ids'].to(device)\n",
        "\n",
        "                loss, logits = model(input_ids = input_ids, attention_mask=mask,\n",
        "                                          labels=train_label, return_dict = False)\n",
        "\n",
        "                total_loss_train += loss.item()\n",
        "\n",
        "                acc = (logits.argmax(dim=1) == train_label).sum().item()\n",
        "                total_acc_train += acc\n",
        "\n",
        "                model.zero_grad()\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "\n",
        "            total_acc_val = 0\n",
        "\n",
        "            with torch.no_grad():\n",
        "                model.eval()\n",
        "                for batch in val_dataloader:\n",
        "\n",
        "                    test_label = batch['labels'].to(device)\n",
        "                    mask = batch['attention_mask'].to(device)\n",
        "                    input_ids = batch['input_ids'].to(device)\n",
        "\n",
        "                    loss, logits = model(input_ids = input_ids, attention_mask=mask,\n",
        "                                          labels=test_label,return_dict = False)\n",
        "\n",
        "                    acc = (logits.argmax(dim=1) == test_label).sum().item()\n",
        "                    total_acc_val += acc\n",
        "\n",
        "            print(\n",
        "                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / train_num: .3f} \\\n",
        "                | Train Accuracy: {total_acc_train / train_num: .3f} \\\n",
        "                | Val Accuracy: {total_acc_val / val_num: .3f}')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "EPOCHS = 10\n",
        "LR = 1e-6\n",
        "train(model, 45000, 5000, LR, EPOCHS, 3, batch = 64)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514,
          "referenced_widgets": [
            "10ddc7c4180e4e09a619bba9df87a291",
            "33ff2f94716b4e0286a7999ae881414a",
            "231bbea44ffe41cb8c523be1e159e48e",
            "37f14f830cdf456cb64598f9c2e97975",
            "ac303bced2594a66bf187bed539382ae",
            "0249b9e428cd48e1a5684d539a2c79af",
            "93a6691533a7450097593a0901d93a3c",
            "dc9fb1754ca44fa18df7da3c9d6819fb",
            "e5bb5b32952c47afa4b7e8aef3e81358",
            "6f72fb7926214eefbc26c74b7c703cc0",
            "b123ceecc0df4c3995845dacbc8f143f",
            "cc1c384765ee44d2b4ca9cf97d7c84d4",
            "2a8ffdd5d5c740f3a14c31d65835b677",
            "55e20724446342b0a9e49425587eff67",
            "862ed75832214a48b6cd9e4bd476757a",
            "7fecb74fbf0a4f3588a20a30c1c9f838",
            "79e78afe7a6a49d8bc2a1339cb608986",
            "408816b23ad6497a9d058c40d1e4447a",
            "ca60f14371184bdda15a5ae11dd2e4d5",
            "68dff87eeb80427d8339a7051e140c44",
            "5e20e4a43beb494bab4c29a4c063bdd0",
            "665d384791f242d0a4c2698399e8c476"
          ]
        },
        "id": "Ky_qcwRgskTu",
        "outputId": "9d74eb6e-222b-4815-b268-e55409faa40d"
      },
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "10ddc7c4180e4e09a619bba9df87a291",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cc1c384765ee44d2b4ca9cf97d7c84d4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/436M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 1 | Train Loss:  0.017                 | Train Accuracy:  0.331                 | Val Accuracy:  0.339\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 704/704 [01:54<00:00,  6.17it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 2 | Train Loss:  0.017                 | Train Accuracy:  0.336                 | Val Accuracy:  0.334\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 3 | Train Loss:  0.017                 | Train Accuracy:  0.337                 | Val Accuracy:  0.322\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 4 | Train Loss:  0.017                 | Train Accuracy:  0.338                 | Val Accuracy:  0.348\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.20it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 5 | Train Loss:  0.017                 | Train Accuracy:  0.337                 | Val Accuracy:  0.322\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 6 | Train Loss:  0.017                 | Train Accuracy:  0.339                 | Val Accuracy:  0.318\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epochs: 7 | Train Loss:  0.017                 | Train Accuracy:  0.338                 | Val Accuracy:  0.335\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.21it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 8 | Train Loss:  0.017                 | Train Accuracy:  0.342                 | Val Accuracy:  0.330\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.20it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 9 | Train Loss:  0.017                 | Train Accuracy:  0.343                 | Val Accuracy:  0.335\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.20it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 10 | Train Loss:  0.017                 | Train Accuracy:  0.340                 | Val Accuracy:  0.334\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#BERT - one gram + (its sum)\n",
        "def generate_model(num_labels):\n",
        "    model = BertForSequenceClassification.from_pretrained('bert-base-cased', num_labels = num_labels)\n",
        "    config = model.config\n",
        "    model = BertForSequenceClassification(config)\n",
        "    return model\n",
        "\n",
        "class integer_dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, num, prime):\n",
        "        self.input_ids = []\n",
        "        self.labels = []\n",
        "        self.attention_mask = []\n",
        "        for i in range(num):\n",
        "            a = random.randint(0, 2**32)\n",
        "            text = [10]\n",
        "            mask = [1]\n",
        "            count = 0\n",
        "            sum = 0\n",
        "            self.labels.append(a%prime)\n",
        "            while a != 0:\n",
        "                text.append(a%10)\n",
        "                sum += a%10\n",
        "                a = a//10\n",
        "                mask.append(1)\n",
        "            text.append(sum)\n",
        "            mask.append(1)\n",
        "            for i in range(13-len(text)):\n",
        "                text.append(10)\n",
        "                mask.append(0)\n",
        "            self.input_ids.append(text)\n",
        "            self.attention_mask.append(mask)\n",
        "        self.input_ids = torch.tensor(self.input_ids)\n",
        "        self.labels = torch.tensor(self.labels)\n",
        "        self.attention_mask = torch.tensor(self.attention_mask)\n",
        "\n",
        "    def classes(self):\n",
        "        return self.labels\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {'input_ids':self.input_ids[idx], 'attention_mask' : self.attention_mask[idx], 'labels':self.labels[idx]}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dataset = integer_dataset(50000, 3)\n",
        "dataloader = torch.utils.data.DataLoader(dataset, batch_size=16)\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "model = generate_model(3)\n",
        "\n",
        "\n",
        "def train(model, train_num, val_num, learning_rate, epochs, prime, batch):\n",
        "\n",
        "    train, val = integer_dataset(train_num,prime), integer_dataset(val_num, prime)\n",
        "\n",
        "    train_dataloader = torch.utils.data.DataLoader(train, batch_size=batch, shuffle=True)\n",
        "    val_dataloader = torch.utils.data.DataLoader(val, batch_size=2)\n",
        "\n",
        "\n",
        "    optimizer = Adam(model.parameters(), lr= learning_rate)\n",
        "    model.train()\n",
        "    if use_cuda:\n",
        "\n",
        "            model = model.cuda()\n",
        "\n",
        "    for epoch_num in range(epochs):\n",
        "\n",
        "            total_acc_train = 0\n",
        "            total_loss_train = 0\n",
        "\n",
        "            for batch in tqdm(train_dataloader):\n",
        "\n",
        "                train_label = batch['labels'].to(device)\n",
        "                mask = batch['attention_mask'].to(device)\n",
        "                input_ids = batch['input_ids'].to(device)\n",
        "\n",
        "                loss, logits = model(input_ids = input_ids, attention_mask=mask,\n",
        "                                          labels=train_label, return_dict = False)\n",
        "\n",
        "                total_loss_train += loss.item()\n",
        "\n",
        "                acc = (logits.argmax(dim=1) == train_label).sum().item()\n",
        "                total_acc_train += acc\n",
        "\n",
        "                model.zero_grad()\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "\n",
        "            total_acc_val = 0\n",
        "\n",
        "            with torch.no_grad():\n",
        "                model.eval()\n",
        "                for batch in val_dataloader:\n",
        "\n",
        "                    test_label = batch['labels'].to(device)\n",
        "                    mask = batch['attention_mask'].to(device)\n",
        "                    input_ids = batch['input_ids'].to(device)\n",
        "\n",
        "                    loss, logits = model(input_ids = input_ids, attention_mask=mask,\n",
        "                                          labels=test_label,return_dict = False)\n",
        "\n",
        "                    acc = (logits.argmax(dim=1) == test_label).sum().item()\n",
        "                    total_acc_val += acc\n",
        "\n",
        "            print(\n",
        "                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / train_num: .3f} \\\n",
        "                | Train Accuracy: {total_acc_train / train_num: .3f} \\\n",
        "                | Val Accuracy: {total_acc_val / val_num: .3f}')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "EPOCHS = 10\n",
        "LR = 1e-6\n",
        "train(model, 45000, 5000, LR, EPOCHS, 3, batch = 64)"
      ],
      "metadata": {
        "id": "I96sefTPtRhM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "497f5d0a-8b79-41e5-fc77-20d99f71cf2f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100%|██████████| 704/704 [01:55<00:00,  6.07it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 1 | Train Loss:  0.012                 | Train Accuracy:  0.655                 | Val Accuracy:  0.994\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.18it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 2 | Train Loss:  0.000                 | Train Accuracy:  0.998                 | Val Accuracy:  0.999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.18it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 3 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.18it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 4 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 5 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 6 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.18it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 7 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 8 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 9 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 10 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#BERT - one gram + (its sum % 3)\n",
        "def generate_model(num_labels):\n",
        "    model = BertForSequenceClassification.from_pretrained('bert-base-cased', num_labels = num_labels)\n",
        "    config = model.config\n",
        "    model = BertForSequenceClassification(config)\n",
        "    return model\n",
        "\n",
        "class integer_dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, num, prime):\n",
        "        self.input_ids = []\n",
        "        self.labels = []\n",
        "        self.attention_mask = []\n",
        "        for i in range(num):\n",
        "            a = random.randint(0, 2**32)\n",
        "            text = [10]\n",
        "            mask = [1]\n",
        "            count = 0\n",
        "            sum = 0\n",
        "            self.labels.append(a%prime)\n",
        "            while a != 0:\n",
        "                text.append(a%10)\n",
        "                sum += a%10\n",
        "                a = a//10\n",
        "                mask.append(1)\n",
        "            text.append(sum % 3)\n",
        "            mask.append(1)\n",
        "            for i in range(13-len(text)):\n",
        "                text.append(10)\n",
        "                mask.append(0)\n",
        "            self.input_ids.append(text)\n",
        "            self.attention_mask.append(mask)\n",
        "        self.input_ids = torch.tensor(self.input_ids)\n",
        "        self.labels = torch.tensor(self.labels)\n",
        "        self.attention_mask = torch.tensor(self.attention_mask)\n",
        "\n",
        "    def classes(self):\n",
        "        return self.labels\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {'input_ids':self.input_ids[idx], 'attention_mask' : self.attention_mask[idx], 'labels':self.labels[idx]}\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dataset = integer_dataset(50000, 3)\n",
        "dataloader = torch.utils.data.DataLoader(dataset, batch_size=16)\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "model = generate_model(3)\n",
        "\n",
        "\n",
        "def train(model, train_num, val_num, learning_rate, epochs, prime, batch):\n",
        "\n",
        "    train, val = integer_dataset(train_num,prime), integer_dataset(val_num, prime)\n",
        "\n",
        "    train_dataloader = torch.utils.data.DataLoader(train, batch_size=batch, shuffle=True)\n",
        "    val_dataloader = torch.utils.data.DataLoader(val, batch_size=2)\n",
        "\n",
        "\n",
        "    optimizer = Adam(model.parameters(), lr= learning_rate)\n",
        "    model.train()\n",
        "    if use_cuda:\n",
        "\n",
        "            model = model.cuda()\n",
        "\n",
        "    for epoch_num in range(epochs):\n",
        "\n",
        "            total_acc_train = 0\n",
        "            total_loss_train = 0\n",
        "\n",
        "            for batch in tqdm(train_dataloader):\n",
        "\n",
        "                train_label = batch['labels'].to(device)\n",
        "                mask = batch['attention_mask'].to(device)\n",
        "                input_ids = batch['input_ids'].to(device)\n",
        "\n",
        "                loss, logits = model(input_ids = input_ids, attention_mask=mask,\n",
        "                                          labels=train_label, return_dict = False)\n",
        "\n",
        "                total_loss_train += loss.item()\n",
        "\n",
        "                acc = (logits.argmax(dim=1) == train_label).sum().item()\n",
        "                total_acc_train += acc\n",
        "\n",
        "                model.zero_grad()\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "\n",
        "            total_acc_val = 0\n",
        "\n",
        "            with torch.no_grad():\n",
        "                model.eval()\n",
        "                for batch in val_dataloader:\n",
        "\n",
        "                    test_label = batch['labels'].to(device)\n",
        "                    mask = batch['attention_mask'].to(device)\n",
        "                    input_ids = batch['input_ids'].to(device)\n",
        "\n",
        "                    loss, logits = model(input_ids = input_ids, attention_mask=mask,\n",
        "                                          labels=test_label,return_dict = False)\n",
        "\n",
        "                    acc = (logits.argmax(dim=1) == test_label).sum().item()\n",
        "                    total_acc_val += acc\n",
        "\n",
        "            print(\n",
        "                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / train_num: .3f} \\\n",
        "                | Train Accuracy: {total_acc_train / train_num: .3f} \\\n",
        "                | Val Accuracy: {total_acc_val / val_num: .3f}')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "EPOCHS = 10\n",
        "LR = 1e-6\n",
        "train(model, 45000, 5000, LR, EPOCHS, 3, batch = 64)"
      ],
      "metadata": {
        "id": "BDcFkjN-uoUC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514,
          "referenced_widgets": [
            "4e75f51cdb114436bb6077039a2bceac",
            "3413d7123c2c455e946d245d9c38a1d9",
            "cbf937c3ef7a4ff7b3c65264ceec2b02",
            "0c5997f4f56e4dffaecbca1a59aaf1ed",
            "22854886768044f3938ef35f36eb4c56",
            "3a88f442f525402f9898ce2638f060e3",
            "3d6d989538a240aa891c8fe33d8233b8",
            "1a78a63da36746ecb4e7776e76ec5843",
            "1324d7d26c414135a0e8933dc04cde1e",
            "c2074af926914628998697cf51a5d5a3",
            "bb95321703f94413b31f1d0fae9f755f",
            "eaeee2077bef446ea0d01ffd139c6bef",
            "a108833940174e9e87b27d0613a7201b",
            "cf367f3b85dd43f3b549f6c7a90489d8",
            "81314df1ccbb47ecbd68de5a1bdf75ff",
            "d1eda5beb978466ca95f72bfb9039594",
            "67b3fb55f97a481fbf11aef380326d53",
            "88962dad58e44a36b63e2ec9e05ec6f4",
            "7285d9af6b044ecebf6196185930afbb",
            "8346bd48676e4ea18432ab4e95e0fd83",
            "1edcc63ae94d4557985d861b2c7a121a",
            "64a50ea0b74f4faab9ab4a4e92047200"
          ]
        },
        "outputId": "300f136b-48cc-4890-91ab-b0b6762daf14"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4e75f51cdb114436bb6077039a2bceac"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/436M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "eaeee2077bef446ea0d01ffd139c6bef"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100%|██████████| 704/704 [01:53<00:00,  6.18it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 1 | Train Loss:  0.007                 | Train Accuracy:  0.798                 | Val Accuracy:  0.993\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 2 | Train Loss:  0.000                 | Train Accuracy:  0.998                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:54<00:00,  6.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 3 | Train Loss:  0.000                 | Train Accuracy:  0.999                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.19it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 4 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  0.999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 5 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  1.000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 6 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  0.999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 7 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  0.999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 8 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  0.999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:52<00:00,  6.23it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 9 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  0.999\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 704/704 [01:53<00:00,  6.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epochs: 10 | Train Loss:  0.000                 | Train Accuracy:  1.000                 | Val Accuracy:  0.999\n"
          ]
        }
      ]
    }
  ]
}