03_Handling_Multiple_Sequences_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Handling Multiple Sequences"
      ],
      "metadata": {
        "id": "3GbmLtPC6Yol"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create batch of inputs and send it to Model"
      ],
      "metadata": {
        "id": "kagZ4zYuJdOf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's convert the list of numbers into a tensor for the sample sequence and send it to the model."
      ],
      "metadata": {
        "id": "CyDRdHBaJlCv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "edvyMwwzHv0F"
      },
      "outputs": [],
      "source": [
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification"
      ],
      "metadata": {
        "id": "2E42J_B2JyDq"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zGxCyGScJ7ff",
        "outputId": "f296d674-dfc7-4710-a6d5-1b2698bde5a0"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sequence = \"I am currently learning machine learning.\""
      ],
      "metadata": {
        "id": "bxhadRUYKU2w"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# create tokens\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "print(tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GkkGAURXKpod",
        "outputId": "5d29a69a-a948-4087-f239-81721e3eb1b3"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['i', 'am', 'currently', 'learning', 'machine', 'learning', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "print(input_ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0a5ZO_ExKxYq",
        "outputId": "21ccecf1-dd46-435e-d27d-e6110cb335c1"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1045, 2572, 2747, 4083, 3698, 4083, 1012]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ids = torch.tensor(input_ids)\n",
        "print(ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ad3OkpQMK6JQ",
        "outputId": "7c107d80-db95-46b2-bf22-3218e67b2e0e"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1045, 2572, 2747, 4083, 3698, 4083, 1012])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model(ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321
        },
        "id": "ZVS9y2EmLDHN",
        "outputId": "9031851f-ee70-4b69-fd47-577a2c5e7bd0"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "too many indices for tensor of dimension 1",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-527d0145ab42>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1532\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1534\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1539\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1540\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1542\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1543\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/distilbert/modeling_distilbert.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    988\u001b[0m         \u001b[0mreturn_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_return_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    989\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 990\u001b[0;31m         distilbert_output = self.distilbert(\n\u001b[0m\u001b[1;32m    991\u001b[0m             \u001b[0minput_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    992\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1530\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1532\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1533\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1534\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1539\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1540\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1541\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1542\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1543\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/distilbert/modeling_distilbert.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    788\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"You cannot specify both input_ids and inputs_embeds at the same time\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    789\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0minput_ids\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 790\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn_if_padding_and_no_attention_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    791\u001b[0m             \u001b[0minput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    792\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0minputs_embeds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mwarn_if_padding_and_no_attention_mask\u001b[0;34m(self, input_ids, attention_mask)\u001b[0m\n\u001b[1;32m   4539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4540\u001b[0m         \u001b[0;31m# Check only the first and last input IDs to reduce overhead.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4541\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad_token_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4542\u001b[0m             warn_string = (\n\u001b[1;32m   4543\u001b[0m                 \u001b[0;34m\"We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: too many indices for tensor of dimension 1"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Why is Model failing?\n",
        "\n",
        "Here, we sent one sequence to the model, and Transformers models require many sentences by default. Let's see how `tokenizer` works by returning `PyTorch` tensor format."
      ],
      "metadata": {
        "id": "RC1Y8Y0JLTAJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_sequence = tokenizer(sequence, return_tensors = 'pt')"
      ],
      "metadata": {
        "id": "cNjDn7BgLJHE"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenized_sequence['input_ids'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b9he_QsAUICY",
        "outputId": "ee5d486a-e32e-4136-e8b7-622fc9cad628"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 101, 1045, 2572, 2747, 4083, 3698, 4083, 1012,  102]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "From the output we got from the tokenizer, we can clearly see that the tokenizer didn't just convert input IDs into tensors, but it added extra dimension on top of it."
      ],
      "metadata": {
        "id": "KQxRxuDmUVui"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's try again by adding a dimension to the list of input IDs."
      ],
      "metadata": {
        "id": "jlJ8l02_UqRm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)"
      ],
      "metadata": {
        "id": "FLxfZQwxULzh"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# create tokens\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "print(tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s3WMp5B_U2uR",
        "outputId": "91708a45-284c-4eda-8f14-dff5a620f47d"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['i', 'am', 'currently', 'learning', 'machine', 'learning', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "print(input_ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1ur3wRP7U8TK",
        "outputId": "b0fc1ef2-1f7f-4c8c-bd75-d04cd8223512"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1045, 2572, 2747, 4083, 3698, 4083, 1012]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ids = torch.tensor([input_ids])\n",
        "print(ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oLHw_5YdU_LH",
        "outputId": "35d8198f-0ac3-4d4d-afd5-ac7f4b496990"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1045, 2572, 2747, 4083, 3698, 4083, 1012]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output = model(ids)\n",
        "print(output.logits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JGO9RbaVVBe5",
        "outputId": "07f8a97d-9a1e-4aa8-d503-43a9d60dfdfc"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-0.6310,  0.7762]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Batching\n",
        "\n",
        "It is the process of sending many sequences to the model at once. We have just built a batch with a single sequence."
      ],
      "metadata": {
        "id": "OZsFzSYXVVcs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "### Padding the sequences\n",
        "\n",
        "`padding` make sure all our sequences have the same length as we make our tensors into rectangles by adding a special token called a `padding token`.\n",
        "\n",
        "The padding token ID can be found in `pad_token_id`.\n",
        "\n",
        "For example, if we have a batch of sequences having input IDs as below:\n",
        "\n",
        "```\n",
        "batch_ids = [\n",
        "              [100, 100, 100],\n",
        "              [100, 100]\n",
        "          ]\n",
        "```\n",
        "\n",
        "Now, let's send our sequences to the model individually and then in the form of batches.\n"
      ],
      "metadata": {
        "id": "FtPiI-XxWeX-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)"
      ],
      "metadata": {
        "id": "poKTvys6XQWY"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "seq1_ids = [[100, 100, 100]]\n",
        "seq2_ids = [[100, 100]]\n",
        "\n",
        "batch_ids = [\n",
        "  [100, 100, 100],\n",
        "  [100, 100, tokenizer.pad_token_id]\n",
        "]"
      ],
      "metadata": {
        "id": "ncCBHBN3VLFE"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(model(torch.tensor(seq1_ids)).logits)\n",
        "print(model(torch.tensor(seq2_ids)).logits)\n",
        "print(model(torch.tensor(batch_ids)).logits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bRBogjdlYG5Y",
        "outputId": "54c1a4c0-1dca-4240-d3b0-210a6b2aa0ac"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 1.4738, -1.3271]], grad_fn=<AddmmBackward0>)\n",
            "tensor([[ 1.2205, -1.1099]], grad_fn=<AddmmBackward0>)\n",
            "tensor([[ 1.4738, -1.3271],\n",
            "        [ 1.7130, -1.4950]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "From the above output, we observe that we got completely different results for our second sequence. It is because our Transformer models take padding tokens into consideration while they attend to all tokens of a sequence.\n",
        "\n",
        "So, we explicitly need to tell attention layers to ignore padding tokens we applied to the sequence for getting the same results for the second sequence passed in a batch or passed individually to the model."
      ],
      "metadata": {
        "id": "h8HE6LZxYuCh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's take an example where we have multiple sequences and we pad them according to multiple objectives:"
      ],
      "metadata": {
        "id": "NgRHprpI2hQG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sequences = [\n",
        "                \"I am learning machine learning\",\n",
        "                \"I am excited to lern new frameworks in ML\"\n",
        "          ]"
      ],
      "metadata": {
        "id": "qM8VQ90u2q_S"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# pad sequences up to maximum sequence length\n",
        "inputs = tokenizer(sequences, padding='longest')\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DVl5RMQ73NGz",
        "outputId": "142f2d35-2641-4580-c2f7-785dc2c069b2"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[101, 1045, 2572, 4083, 3698, 4083, 102, 0, 0, 0, 0, 0, 0], [101, 1045, 2572, 7568, 2000, 3393, 6826, 2047, 7705, 2015, 1999, 19875, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pad sequences up to model maximum length (512 for BERT)\n",
        "inputs = tokenizer(sequences, padding='max_length')\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fTV_FcxL3NEb",
        "outputId": "14ef9fd1-156b-4344-96d7-18e634d93d4f"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[101, 1045, 2572, 4083, 3698, 4083, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1045, 2572, 7568, 2000, 3393, 6826, 2047, 7705, 2015, 1999, 19875, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pad sequences up to specified maximum length\n",
        "inputs = tokenizer(sequences, padding='max_length', max_length=8)\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4qSiuZss3NB0",
        "outputId": "2a5cb898-d24f-4f8c-a05b-1a6529e46517"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[101, 1045, 2572, 4083, 3698, 4083, 102, 0], [101, 1045, 2572, 7568, 2000, 3393, 6826, 2047, 7705, 2015, 1999, 19875, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can also trucate sequences"
      ],
      "metadata": {
        "id": "g0Dr_il-4Bv0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# truncate sequences that are longer than the model max length\n",
        "inputs = tokenizer(\n",
        "                        sequences,\n",
        "                        truncation=True\n",
        "                    )\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w1lp07_G3M-y",
        "outputId": "56252955-599a-44f0-809b-30e38ef56ec1"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[101, 1045, 2572, 4083, 3698, 4083, 102], [101, 1045, 2572, 7568, 2000, 3393, 6826, 2047, 7705, 2015, 1999, 19875, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# truncate sequences that are longer than the specified max length\n",
        "inputs = tokenizer(\n",
        "                        sequences,\n",
        "                        max_length=8,\n",
        "                        truncation=True\n",
        "                    )\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PlD6PUPs4aTB",
        "outputId": "caa2f5fd-b41b-4597-c571-01bec7a8d46b"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': [[101, 1045, 2572, 4083, 3698, 4083, 102], [101, 1045, 2572, 7568, 2000, 3393, 6826, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Moreover, we can utilize the `tokenizer` object to return tensors from different frameworks, like\n",
        "\n",
        "* `pt` return PyTorch tensors\n",
        "* `tf` return Tensorflow tensors\n",
        "* `np` returns NumPy tensors.\n"
      ],
      "metadata": {
        "id": "X65VEkxt4yL7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# return Pytorch tensors\n",
        "inputs = tokenizer(sequences, padding=True, return_tensors='pt')\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7GOVDOy65MeO",
        "outputId": "47412b76-210c-4f60-ba49-1143a3b5f48a"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': tensor([[  101,  1045,  2572,  4083,  3698,  4083,   102,     0,     0,     0,\n",
            "             0,     0,     0],\n",
            "        [  101,  1045,  2572,  7568,  2000,  3393,  6826,  2047,  7705,  2015,\n",
            "          1999, 19875,   102]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
            "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# return tensorflow tensors\n",
        "inputs = tokenizer(sequences, padding=True, return_tensors='tf')\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kOyuA4Ky5ZVa",
        "outputId": "10709477-d7e3-4edf-cbaa-2b97262a3225"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': <tf.Tensor: shape=(2, 13), dtype=int32, numpy=\n",
            "array([[  101,  1045,  2572,  4083,  3698,  4083,   102,     0,     0,\n",
            "            0,     0,     0,     0],\n",
            "       [  101,  1045,  2572,  7568,  2000,  3393,  6826,  2047,  7705,\n",
            "         2015,  1999, 19875,   102]], dtype=int32)>, 'attention_mask': <tf.Tensor: shape=(2, 13), dtype=int32, numpy=\n",
            "array([[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
            "       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=int32)>}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# return numpy tensors\n",
        "inputs = tokenizer(sequences, padding=True, return_tensors='np')\n",
        "print(inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lfpwa2Cc5dKs",
        "outputId": "e935639f-1573-401a-faab-50a84a6e6d13"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': array([[  101,  1045,  2572,  4083,  3698,  4083,   102,     0,     0,\n",
            "            0,     0,     0,     0],\n",
            "       [  101,  1045,  2572,  7568,  2000,  3393,  6826,  2047,  7705,\n",
            "         2015,  1999, 19875,   102]]), 'attention_mask': array([[1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
            "       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Attention masks"
      ],
      "metadata": {
        "id": "Pi0uuen5ZxzV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Attention masks** are defined as tensors with a value of 1s or 0s, where,\n",
        "\n",
        "\n",
        "* 1s indicate the token should be attended to, and\n",
        "* 0s represent the token should not be attended to."
      ],
      "metadata": {
        "id": "HBU2VngqZ7MB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's see how we use attention for our above example"
      ],
      "metadata": {
        "id": "ABg7eCskbH_1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_ids = [\n",
        "  [100, 100, 100],\n",
        "  [100, 100, tokenizer.pad_token_id]\n",
        "]\n",
        "\n",
        "attention_mask = [\n",
        "    [1, 1, 1],\n",
        "    [1, 1, 0]\n",
        "]"
      ],
      "metadata": {
        "id": "4Hjg-ZYPYYIj"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "outputs = model(\n",
        "                    torch.tensor(batch_ids),\n",
        "                    attention_mask = torch.tensor(attention_mask)\n",
        "                )"
      ],
      "metadata": {
        "id": "vhogEFqMbavA"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(outputs.logits)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eo5u3I8WbuiL",
        "outputId": "50bb031f-8a8e-4797-d853-9f5529c0bc2c"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 1.4738, -1.3271],\n",
            "        [ 1.2205, -1.1099]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, we have the same logits for the second sequence as we expected to see."
      ],
      "metadata": {
        "id": "_MZIup_db0Fz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Longer sequence"
      ],
      "metadata": {
        "id": "5NEsQR-LcMyo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Transformer models can hadle sequence length up to 512 or 1024 tokens and will crash if applied longer sequence length.\n",
        "\n",
        "\n",
        "**Solution**:\n",
        "\n",
        "* Use model with longer supported sequence length, or\n",
        "* Truncate the sequence length."
      ],
      "metadata": {
        "id": "GRcgdb3ccSuW"
      }
    }
  ]
}