fine-tune-clip.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qaBYR9UAxF7h"
      },
      "source": [
        "# Setup \n",
        "\n",
        "* Installs necessary packages \n",
        "\n",
        "* Downloads data to local storage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rbOnouxSp57X",
        "outputId": "b456aa7a-648a-4ab6-8284-9810adfbf364"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.21.1-py3-none-any.whl (4.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.7 MB 31.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 54.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.9.0-py3-none-any.whl (120 kB)\n",
            "\u001b[K     |████████████████████████████████| 120 kB 74.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.12.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.6.15)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.9.0 tokenizers-0.12.1 transformers-4.21.1\n",
            "--2022-08-24 11:05:54--  http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n",
            "Resolving images.cocodataset.org (images.cocodataset.org)... 52.217.135.73\n",
            "Connecting to images.cocodataset.org (images.cocodataset.org)|52.217.135.73|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 252907541 (241M) [application/zip]\n",
            "Saving to: ‘annotations_trainval2017.zip’\n",
            "\n",
            "annotations_trainva 100%[===================>] 241.19M  12.4MB/s    in 22s     \n",
            "\n",
            "2022-08-24 11:06:16 (11.2 MB/s) - ‘annotations_trainval2017.zip’ saved [252907541/252907541]\n",
            "\n",
            "Archive:  annotations_trainval2017.zip\n",
            "  inflating: annotations/instances_train2017.json  \n",
            "  inflating: annotations/instances_val2017.json  \n",
            "  inflating: annotations/captions_train2017.json  \n",
            "  inflating: annotations/captions_val2017.json  \n",
            "  inflating: annotations/person_keypoints_train2017.json  \n",
            "  inflating: annotations/person_keypoints_val2017.json  \n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!rm -r *\n",
        "!wget http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n",
        "!unzip annotations_trainval2017.zip\n",
        "!rm -r annotations_trainval2017.zip\n",
        "!rm -r annotations/instances_train2017.json\n",
        "!rm -r annotations/instances_val2017.json\n",
        "!rm -r annotations/person_keypoints_train2017.json\n",
        "!rm -r annotations/person_keypoints_val2017.json"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xeWyMoszwCl9"
      },
      "source": [
        "# Imports\n",
        "\n",
        "* Importation of necessary packages for the following code to work."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vRvrWdOrpFMY"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from transformers import logging\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "import requests\n",
        "from transformers import (\n",
        "    VisionTextDualEncoderModel,\n",
        "    VisionTextDualEncoderProcessor,\n",
        "    ViTFeatureExtractor,\n",
        "    BertTokenizer,\n",
        ")\n",
        "logging.set_verbosity_error()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dwhKP7SHxaGI"
      },
      "source": [
        "# Configuration \n",
        "* Choice of Image & Text encoders\n",
        "* Choice of learning params\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h-QXE4sfxccI"
      },
      "outputs": [],
      "source": [
        "class CFG:\n",
        "\n",
        "    max_text_tokens_length = 128\n",
        "    text_backbone = 'bert-base-uncased'\n",
        "    image_backbone = 'google/vit-base-patch16-224'\n",
        "\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    batch_size = 32\n",
        "    max_epochs = 75\n",
        "    max_bad_epochs = 9\n",
        "    patience = 3\n",
        "    factor = 0.1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eaPobnxdv_SF"
      },
      "source": [
        "# Functions\n",
        "\n",
        "* Definition of COCO dataset loading methods\n",
        "* Definition of Learning methods "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TYKoHEecpUHn"
      },
      "outputs": [],
      "source": [
        "def read_coco_pairs(split='train'):\n",
        "  if split == 'train':\n",
        "    path = 'annotations/captions_train2017.json'\n",
        "  else:\n",
        "    path = 'annotations/captions_val2017.json'\n",
        "  with open(path, 'r') as file:\n",
        "      data = json.load(file)\n",
        "  images = list(map(lambda x: (x['id'], x['flickr_url']), data['images']))\n",
        "  images = {x[0]: x[1] for x in images}\n",
        "  annotations = list(map(lambda x:(x['image_id'], x['caption']),data['annotations']))\n",
        "  pairs = list(map(lambda x:{'caption':x[1],'url':images[x[0]]}, annotations))\n",
        "  return pairs\n",
        "\n",
        "class DataSet(torch.utils.data.Dataset):\n",
        "\n",
        "  def __init__(self, pairs, processor):\n",
        "    super().__init__()\n",
        "    self.pairs = pairs\n",
        "    self.processor = processor\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    try:\n",
        "      caption = self.pairs[idx]['caption']\n",
        "      url = self.pairs[idx]['url']\n",
        "      image = Image.open(requests.get(url, stream=True).raw) \n",
        "      encoded_pair = self.processor(text=[caption], images=[image], return_tensors=\"pt\", max_length=CFG.max_text_tokens_length, padding='max_length', truncation=True)\n",
        "      return encoded_pair\n",
        "    except:\n",
        "      return None\n",
        "  \n",
        "  def __len__(self):\n",
        "    return len(self.pairs)\n",
        "\n",
        "def collate_fn(batch):\n",
        "  batch = list(filter(lambda x: x is not None, batch))\n",
        "  return torch.utils.data.dataloader.default_collate(batch)\n",
        "\n",
        "def train_epoch(model, train_loader, optimizer, epoch, max_epochs):\n",
        "    model.train()\n",
        "    nb_batches = len(train_loader)\n",
        "    tqdm_object = tqdm(train_loader, total=len(train_loader))   \n",
        "    epoch_loss = 0.0\n",
        "    for i, batch in enumerate(tqdm_object):\n",
        "      outputs = model(\n",
        "          input_ids=batch['input_ids'].squeeze().to(CFG.device),\n",
        "          attention_mask=batch['attention_mask'].squeeze().to(CFG.device),\n",
        "          pixel_values=batch['pixel_values'].squeeze().to(CFG.device),\n",
        "          return_loss=True)\n",
        "      loss, logits_per_image = outputs.loss, outputs.logits_per_image  # this is the image-text similarity score\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      tqdm_object.set_postfix(\n",
        "          batch=\"{}/{}\".format(i+1,nb_batches),\n",
        "          train_loss=loss.item(),\n",
        "          lr=get_lr(optimizer)\n",
        "          )\n",
        "    epoch_loss = epoch_loss / nb_batches\n",
        "    return epoch_loss\n",
        "\n",
        "def valid_epoch(model, dev_loader, epoch, max_epochs):\n",
        "    model.eval()\n",
        "    nb_batches = len(dev_loader)\n",
        "    tqdm_object = tqdm(dev_loader, total=len(dev_loader))\n",
        "    epoch_loss = 0.0   \n",
        "    for i, batch in enumerate(tqdm_object):\n",
        "      outputs = model(\n",
        "          input_ids=batch['input_ids'].squeeze(),\n",
        "          attention_mask=batch['attention_mask'].squeeze(),\n",
        "          pixel_values=batch['pixel_values'].squeeze(),\n",
        "          return_loss=True)\n",
        "      loss, logits_per_image = outputs.loss, outputs.logits_per_image  # this is the image-text similarity score\n",
        "      epoch_loss += loss.item()\n",
        "      tqdm_object.set_postfix(\n",
        "          batch=\"{}/{}\".format(i+1,nb_batches),\n",
        "          dev_loss=loss.item(),\n",
        "          )\n",
        "    epoch_loss = epoch_loss / nb_batches\n",
        "    return epoch_loss\n",
        "\n",
        "def learning_loop(model):\n",
        "    model.to(CFG.device)\n",
        "    optimizer = torch.optim.AdamW(model.parameters(), weight_decay=0.)\n",
        "    lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode=\"min\", patience=CFG.patience, factor=CFG.factor)\n",
        "\n",
        "    best_dev_score = float('inf')\n",
        "    train_history = []\n",
        "    dev_history = []\n",
        "    nb_bad_epochs = 0\n",
        "\n",
        "    print(\"Learning phase\")\n",
        "    print('Used device:', CFG.device)\n",
        "    print(\"--------------\")\n",
        "    for epoch in range(1, CFG.max_epochs+1):\n",
        "\n",
        "        print(\"Epoch {:03d}/{:03d}\".format(epoch, CFG.max_epochs))\n",
        "\n",
        "        if nb_bad_epochs >= CFG.max_bad_epochs:\n",
        "            print(\"Epoch {:03d}/{:03d}: exiting training after too many bad epochs.\".format(epoch, CFG.max_epochs))\n",
        "            torch.save(model.state_dict(), \"final.pt\")\n",
        "            break\n",
        "\n",
        "        else:\n",
        "\n",
        "            epoch_start_time = time.time()\n",
        "\n",
        "            epoch_train_loss = train_epoch(model=model, train_loader=train_dataloader, optimizer=optimizer, epoch=epoch, max_epochs=CFG.max_epochs)\n",
        "            epoch_dev_score = valid_epoch(model=model, dev_loader=val_dataloader, epoch=epoch, max_epochs=CFG.max_epochs)\n",
        "\n",
        "            duration = time.time() - epoch_start_time\n",
        "\n",
        "            lr_scheduler.step(epoch_dev_score)\n",
        "\n",
        "            train_history.append(epoch_train_loss)\n",
        "            dev_history.append(epoch_dev_score)\n",
        "\n",
        "            if epoch_dev_score < best_dev_score:\n",
        "                nb_bad_epochs = 0\n",
        "                best_dev_score = epoch_dev_score\n",
        "                torch.save(model.state_dict(), \"best.pt\")\n",
        "                print(\"Finished epoch {:03d}/{:03d} - Train loss: {:.7f} - Valid loss: {:.7f} - SAVED (NEW) BEST MODEL. Duration: {:.3f} s\".format(\n",
        "                epoch, CFG.max_epochs, epoch_train_loss, epoch_dev_score, duration))\n",
        "            else:\n",
        "                nb_bad_epochs += 1\n",
        "                print(\"Finished epoch {:03d}/{:03d} - Train loss: {:.7f} - Valid loss: {:.7f} - NUMBER OF BAD EPOCH.S: {}. Duration: {:.3f} s\".format(\n",
        "                epoch, CFG.max_epochs, epoch_train_loss, epoch_dev_score, nb_bad_epochs, duration))\n",
        "    \n",
        "    history = {'train':train_history,'dev':dev_history}\n",
        "    return history\n",
        "  \n",
        "def get_lr(optimizer):\n",
        "    for param_group in optimizer.param_groups:\n",
        "        return param_group[\"lr\"]\n",
        "\n",
        "def plot_history(history):\n",
        "    train_history = history['train']\n",
        "    dev_history = history['dev']\n",
        "    plt.plot(list(range(1, len(train_history)+1)), train_history, label=\"train loss\")\n",
        "    plt.plot(list(range(1, len(dev_history)+1)), dev_history, label=\"dev loss\")\n",
        "    plt.xticks(list(range(1, len(train_history)+1)))\n",
        "    plt.xlabel(\"epoch\")\n",
        "    plt.legend()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B2EAL_u3RTDo"
      },
      "source": [
        "# Initializing a data processor\n",
        "\n",
        "* Init. of a text Tokenizer\n",
        "* Init. of an image Feature extractor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181,
          "referenced_widgets": [
            "52187321095648d4ad6620622dafc64c",
            "8a3214c228a9440694483d2bf0c20001",
            "58a546227c3f4c0fa5ec7708bc7831ce",
            "db915068c1a94fb58e973950ef488b01",
            "eb766c07ce7049d6b599e917353163bd",
            "767a03b825214753806163b5ed60012f",
            "d08d8f165cf5472b8d364512ac52a99a",
            "91db86bc84e54280ab9a7facdf3c4f78",
            "31af529b33764764bad0db0c4e49a6a0",
            "774345caaec24c8fb6c419d2a271747e",
            "7044b2cddcb44b179308b055273635b6",
            "47148701c06b40faa4ea56a884741f4b",
            "3fdb54e131264956bfe1305968dcf875",
            "1d4492ec9ab645ca8da160d6ae636cfb",
            "8630d4aa76834368afc60b26cf6ba86c",
            "c9d43a471b294c6fbb58ffa82dba8a2b",
            "12be379cfd4d4804b9d9f535178f221f",
            "c4df91cb91134e308353f3cc812fb777",
            "a0401a0d8864410c8659bce4e8bc6728",
            "85e61da92bcd4c7f8322c92d61772a73",
            "6a00acc1cdcd45089bde2c922ffa6792",
            "26e18b2ab28d48f5bef8b112dd630ea2",
            "61adc0c1c4784798bdefbe0acb238b0f",
            "4f7561aaedac4db3a46c7f21322edcf6",
            "bd5cbc3ebb224310a4c51e402c5b8d27",
            "91852e00df064dafa2038ea95b4de287",
            "547cc055c1174928bb20ba5bf4710ff4",
            "e6a53e34ad8640c5b24bd285f6a73e73",
            "665b1f180abd4b25bbddf00d3170ce2f",
            "70e940adaf9841b3a5c5bb6167185eed",
            "0a458030ee964ef3b4a560fa5293ec58",
            "305165bc3db249039e6c612a2c52599e",
            "9fc9c3a3ff984003a2315b780a73260a",
            "1295c8e5925943edb4c94eca77008bf6",
            "836c4c3163fd4438a403530216fbc955",
            "49bf7f36a92d4a039d650eb12337c3d8",
            "fa67afadc6fa4dc591f04ead07cfdb0a",
            "00f38da5a3bc43e2be9cd64c0eba884b",
            "582143ffca4f46a080a993401b16379b",
            "61d19b93e62e40a9928ad6ab69c1238c",
            "47e9d560728d452e9ed76e43faa105b9",
            "83eec8d183cd450ca7d551003a5bfe81",
            "a5dd39c2a2254deba85f31e053d3a65c",
            "c1dc75bfe3e14e7782843ee5635b2769"
          ]
        },
        "id": "OHPZFKmWRozO",
        "outputId": "25a07bd4-f7f6-4974-8384-e6566eca8f2f"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "52187321095648d4ad6620622dafc64c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading vocab.txt:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "47148701c06b40faa4ea56a884741f4b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61adc0c1c4784798bdefbe0acb238b0f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1295c8e5925943edb4c94eca77008bf6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading preprocessor_config.json:   0%|          | 0.00/160 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CPU times: user 500 ms, sys: 38 ms, total: 538 ms\n",
            "Wall time: 11.4 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "feature_extractor = ViTFeatureExtractor.from_pretrained(\"google/vit-base-patch16-224\")\n",
        "processor = VisionTextDualEncoderProcessor(feature_extractor, tokenizer)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gLExxD5gwTgm"
      },
      "source": [
        "# Creating torch dataset\n",
        "\n",
        "* Loading COCO (text, image) pairs  \n",
        "* Wrapping pairs into a torch dataset, with the data processor attached "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WMqYewJ5NmjV",
        "outputId": "cfb10567-240a-4057-a959-47af471bf1ca"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CPU times: user 2.12 s, sys: 437 ms, total: 2.56 s\n",
            "Wall time: 3.46 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "train_pairs = read_coco_pairs(split='train')\n",
        "val_pairs = read_coco_pairs(split='val')\n",
        "train_ds = DataSet(pairs=train_pairs, processor=processor)\n",
        "dev_ds = DataSet(pairs=val_pairs, processor=processor)\n",
        "train_dataloader = torch.utils.data.DataLoader(train_ds, collate_fn=collate_fn, batch_size=CFG.batch_size)\n",
        "val_dataloader = torch.utils.data.DataLoader(dev_ds, collate_fn=collate_fn, batch_size=CFG.batch_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DJr2nu_kwXRb"
      },
      "source": [
        "# Initializing CLIP model \n",
        "\n",
        "* Instantiating a new CLIP model, using prescribed text & image pre-trained backbones."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OeRXNmFGprIF"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "clip = VisionTextDualEncoderModel.from_vision_text_pretrained(CFG.image_backbone, CFG.text_backbone)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l59s7zeSwxsK"
      },
      "source": [
        "# Learning (fine-tuning) loop\n",
        "\n",
        "* Updating weights of pre-trained encoders and projection head while minimizing constrastive loss."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "dfNrvRcyptNQ",
        "outputId": "73ab47f6-556e-41eb-ec97-1f61cfa95e85"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Learning phase\n",
            "--------------\n",
            "Used device: cuda\n",
            "Epoch 001/075\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 13%|█▎        | 2441/18493 [3:29:54<19:26:38,  4.36s/it, batch=2441/18493, lr=0.001, train_loss=3.18]"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "history = learning_loop(clip)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sT8I1ZKjwiy0"
      },
      "source": [
        "# Learning history\n",
        "\n",
        "* Visualizing learning curves"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RqcYaJswpvK1"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "plot_history(history)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6-DfnfrGcGSI"
      },
      "source": [
        "# Testing"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "qaBYR9UAxF7h",
        "xeWyMoszwCl9",
        "dwhKP7SHxaGI",
        "eaPobnxdv_SF",
        "B2EAL_u3RTDo",
        "gLExxD5gwTgm",
        "DJr2nu_kwXRb",
        "sT8I1ZKjwiy0",
        "6-DfnfrGcGSI"
      ],
      "name": "cliplike-fixed-device.ipynb",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00f38da5a3bc43e2be9cd64c0eba884b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a458030ee964ef3b4a560fa5293ec58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1295c8e5925943edb4c94eca77008bf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_836c4c3163fd4438a403530216fbc955",
              "IPY_MODEL_49bf7f36a92d4a039d650eb12337c3d8",
              "IPY_MODEL_fa67afadc6fa4dc591f04ead07cfdb0a"
            ],
            "layout": "IPY_MODEL_00f38da5a3bc43e2be9cd64c0eba884b"
          }
        },
        "12be379cfd4d4804b9d9f535178f221f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d4492ec9ab645ca8da160d6ae636cfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a0401a0d8864410c8659bce4e8bc6728",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_85e61da92bcd4c7f8322c92d61772a73",
            "value": 28
          }
        },
        "26e18b2ab28d48f5bef8b112dd630ea2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "305165bc3db249039e6c612a2c52599e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31af529b33764764bad0db0c4e49a6a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3fdb54e131264956bfe1305968dcf875": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12be379cfd4d4804b9d9f535178f221f",
            "placeholder": "​",
            "style": "IPY_MODEL_c4df91cb91134e308353f3cc812fb777",
            "value": "Downloading tokenizer_config.json: 100%"
          }
        },
        "47148701c06b40faa4ea56a884741f4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3fdb54e131264956bfe1305968dcf875",
              "IPY_MODEL_1d4492ec9ab645ca8da160d6ae636cfb",
              "IPY_MODEL_8630d4aa76834368afc60b26cf6ba86c"
            ],
            "layout": "IPY_MODEL_c9d43a471b294c6fbb58ffa82dba8a2b"
          }
        },
        "47e9d560728d452e9ed76e43faa105b9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49bf7f36a92d4a039d650eb12337c3d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47e9d560728d452e9ed76e43faa105b9",
            "max": 160,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_83eec8d183cd450ca7d551003a5bfe81",
            "value": 160
          }
        },
        "4f7561aaedac4db3a46c7f21322edcf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e6a53e34ad8640c5b24bd285f6a73e73",
            "placeholder": "​",
            "style": "IPY_MODEL_665b1f180abd4b25bbddf00d3170ce2f",
            "value": "Downloading config.json: 100%"
          }
        },
        "52187321095648d4ad6620622dafc64c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8a3214c228a9440694483d2bf0c20001",
              "IPY_MODEL_58a546227c3f4c0fa5ec7708bc7831ce",
              "IPY_MODEL_db915068c1a94fb58e973950ef488b01"
            ],
            "layout": "IPY_MODEL_eb766c07ce7049d6b599e917353163bd"
          }
        },
        "547cc055c1174928bb20ba5bf4710ff4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "582143ffca4f46a080a993401b16379b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58a546227c3f4c0fa5ec7708bc7831ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_91db86bc84e54280ab9a7facdf3c4f78",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_31af529b33764764bad0db0c4e49a6a0",
            "value": 231508
          }
        },
        "61adc0c1c4784798bdefbe0acb238b0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f7561aaedac4db3a46c7f21322edcf6",
              "IPY_MODEL_bd5cbc3ebb224310a4c51e402c5b8d27",
              "IPY_MODEL_91852e00df064dafa2038ea95b4de287"
            ],
            "layout": "IPY_MODEL_547cc055c1174928bb20ba5bf4710ff4"
          }
        },
        "61d19b93e62e40a9928ad6ab69c1238c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "665b1f180abd4b25bbddf00d3170ce2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a00acc1cdcd45089bde2c922ffa6792": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7044b2cddcb44b179308b055273635b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "70e940adaf9841b3a5c5bb6167185eed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "767a03b825214753806163b5ed60012f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "774345caaec24c8fb6c419d2a271747e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "836c4c3163fd4438a403530216fbc955": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_582143ffca4f46a080a993401b16379b",
            "placeholder": "​",
            "style": "IPY_MODEL_61d19b93e62e40a9928ad6ab69c1238c",
            "value": "Downloading preprocessor_config.json: 100%"
          }
        },
        "83eec8d183cd450ca7d551003a5bfe81": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "85e61da92bcd4c7f8322c92d61772a73": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8630d4aa76834368afc60b26cf6ba86c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a00acc1cdcd45089bde2c922ffa6792",
            "placeholder": "​",
            "style": "IPY_MODEL_26e18b2ab28d48f5bef8b112dd630ea2",
            "value": " 28.0/28.0 [00:00&lt;00:00, 293B/s]"
          }
        },
        "8a3214c228a9440694483d2bf0c20001": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_767a03b825214753806163b5ed60012f",
            "placeholder": "​",
            "style": "IPY_MODEL_d08d8f165cf5472b8d364512ac52a99a",
            "value": "Downloading vocab.txt: 100%"
          }
        },
        "91852e00df064dafa2038ea95b4de287": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_305165bc3db249039e6c612a2c52599e",
            "placeholder": "​",
            "style": "IPY_MODEL_9fc9c3a3ff984003a2315b780a73260a",
            "value": " 570/570 [00:00&lt;00:00, 7.76kB/s]"
          }
        },
        "91db86bc84e54280ab9a7facdf3c4f78": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9fc9c3a3ff984003a2315b780a73260a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0401a0d8864410c8659bce4e8bc6728": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5dd39c2a2254deba85f31e053d3a65c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd5cbc3ebb224310a4c51e402c5b8d27": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70e940adaf9841b3a5c5bb6167185eed",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0a458030ee964ef3b4a560fa5293ec58",
            "value": 570
          }
        },
        "c1dc75bfe3e14e7782843ee5635b2769": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4df91cb91134e308353f3cc812fb777": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9d43a471b294c6fbb58ffa82dba8a2b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d08d8f165cf5472b8d364512ac52a99a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "db915068c1a94fb58e973950ef488b01": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_774345caaec24c8fb6c419d2a271747e",
            "placeholder": "​",
            "style": "IPY_MODEL_7044b2cddcb44b179308b055273635b6",
            "value": " 226k/226k [00:00&lt;00:00, 238kB/s]"
          }
        },
        "e6a53e34ad8640c5b24bd285f6a73e73": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb766c07ce7049d6b599e917353163bd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa67afadc6fa4dc591f04ead07cfdb0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a5dd39c2a2254deba85f31e053d3a65c",
            "placeholder": "​",
            "style": "IPY_MODEL_c1dc75bfe3e14e7782843ee5635b2769",
            "value": " 160/160 [00:00&lt;00:00, 1.67kB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}