conv1dtests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a88fdfcf-3cd4-4bf3-a3de-361af74202f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn import Conv1d\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "id": "f00e43da-3bfc-4af2-8923-5e2a15329690",
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_dim=8\n",
    "conv_bias = False\n",
    "d_conv = 8\n",
    "conv1d = Conv1d(\n",
    "            in_channels=conv_dim,\n",
    "            out_channels=conv_dim,\n",
    "            bias=conv_bias,\n",
    "            kernel_size=d_conv,\n",
    "            groups=conv_dim,\n",
    "            padding=d_conv - 1,\n",
    "            #**factory_kwargs,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "id": "f1bf6247-8926-4dc7-a289-d8426f748301",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Conv1d(8, 8, kernel_size=(8,), stride=(1,), padding=(7,), groups=8, bias=False)"
      ]
     },
     "execution_count": 463,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv1d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "id": "e7ca3242-2581-4809-80d4-445c64e9dce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "size = 1024\n",
    "assert size%8 ==0\n",
    "data = torch.randn(4,conv_dim,size)\n",
    "whole_res = conv1d(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "id": "840f9f90-9f84-47a3-b417-f574e9643b39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 8, 1031])"
      ]
     },
     "execution_count": 481,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "whole_res.shape\n",
    "#Total output size is d_conv-1 = (2*padding -(kernel -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "id": "7b1d4ae9-5f75-4adf-9046-6b73352c4c71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 135])\n",
      "torch.Size([4, 8, 128])\n"
     ]
    }
   ],
   "source": [
    "from einops import rearrange\n",
    "s = size//8 #subsequence length\n",
    "padding = d_conv - 1\n",
    "print(padding)\n",
    "data_split = [data[:,:,max(0,s*i):s*(i+1)+padding] for i in range(data.shape[-1]//s)]\n",
    "#data_split = rearrange(data, 'i j (n k) -> n i j k', n = 2)\n",
    "#print(1*s+conv_dim)\n",
    "#print(data_split[1].shape)\n",
    "for d in data_split:\n",
    "    print(d.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "id": "e07a3b15-080b-4d58-ae59-646f3b27d18a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 142])\n",
      "torch.Size([4, 8, 135])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "98"
      ]
     },
     "execution_count": 483,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [conv1d(x) for x in data_split]\n",
    "for r in res:\n",
    "    print(r.shape)\n",
    "torch.cat(res,dim=-1).shape[-1]-whole_res.shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "id": "b9d7fb2b-70e7-4a8b-b45b-5105d6cf4fd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([  0.6091,   1.5338,   3.0116,   1.6812,   5.2154,   2.9534,   2.6096,\n",
      "          2.0200,  -1.1160,  -1.4831,  -0.7726,  -0.4137,  -3.5267,   0.4202,\n",
      "         -5.7768,  -0.3872,   0.4537,  -1.2695,  -2.0984,   0.3771,  -1.5114,\n",
      "          1.2686,   1.5624,   1.7864,   3.7400,  -1.4852,   3.9187,  -2.3934,\n",
      "          0.7080,   5.2669,  -0.0904,  -1.1988,   1.7409,  -0.4490,  -0.5236,\n",
      "         -2.4334,   5.0603,   1.1493,  -4.1790,   0.5379,  -9.3268,  -2.4407,\n",
      "         -2.3760,  -7.7185,  -6.8241,   2.1141,  -2.3018,   1.5181,   0.8638,\n",
      "          3.2186,   3.4284,   6.1050,   1.9847,   6.3326,   9.1529,   3.3170,\n",
      "          7.7066,   3.6997,   2.6323,   0.1008,  -3.0512,  -0.3031,   5.6420,\n",
      "          5.1077,  -0.2889,   1.2248,  -5.6695,   4.3524,   5.5425,  -6.0713,\n",
      "         -9.3048,  -3.8791,  -2.6624,  -5.2890,  -6.5840, -12.8727,  -1.8465,\n",
      "          1.0117,  -7.3475,  -1.8230,  -1.0675,  -0.9378,   6.0332,  -2.9050,\n",
      "         -5.4279,  -0.3289,  -7.4438,   1.2123,   0.2799,   3.8951,  -0.7419,\n",
      "          2.9967,  -1.9475,   0.3669,   4.1705,   0.4131,  -0.8882,   0.9231,\n",
      "         -0.5112,  -4.2923,   4.6850,   4.2909,  -5.3442,   3.6022,   0.8800,\n",
      "         -3.3514,  -1.1483,   0.7188,  -1.4102,  -0.0244,   2.8213,  -0.3856,\n",
      "         -3.0411,  -3.1663,  -5.0189,  -3.6008,  -2.8663,   0.1998,   4.6103,\n",
      "         -4.7194,  -4.7101,  -0.3997,  -3.9943,   1.7160,  -0.0153,   3.2959,\n",
      "         -2.6514,   1.6684,   1.3652,  -1.5603,   0.4679,  -0.5590,  -1.0911,\n",
      "         -0.9756,  -0.3755], grad_fn=<SliceBackward0>)\n",
      "tensor([  0.6091,   1.5338,   3.0116,   1.6812,   5.2154,   2.9534,   2.6096,\n",
      "          2.0200,  -1.1160,  -1.4831,  -0.7726,  -0.4137,  -3.5267,   0.4202,\n",
      "         -5.7768,  -0.3872,   0.4537,  -1.2695,  -2.0984,   0.3771,  -1.5114,\n",
      "          1.2686,   1.5624,   1.7864,   3.7400,  -1.4852,   3.9187,  -2.3934,\n",
      "          0.7080,   5.2669,  -0.0904,  -1.1988,   1.7409,  -0.4490,  -0.5236,\n",
      "         -2.4334,   5.0603,   1.1493,  -4.1790,   0.5379,  -9.3268,  -2.4407,\n",
      "         -2.3760,  -7.7185,  -6.8241,   2.1141,  -2.3018,   1.5181,   0.8638,\n",
      "          3.2186,   3.4284,   6.1050,   1.9847,   6.3326,   9.1529,   3.3170,\n",
      "          7.7066,   3.6997,   2.6323,   0.1008,  -3.0512,  -0.3031,   5.6420,\n",
      "          5.1077,  -0.2889,   1.2248,  -5.6695,   4.3524,   5.5425,  -6.0713,\n",
      "         -9.3048,  -3.8791,  -2.6624,  -5.2890,  -6.5840, -12.8727,  -1.8465,\n",
      "          1.0117,  -7.3475,  -1.8230,  -1.0675,  -0.9378,   6.0332,  -2.9050,\n",
      "         -5.4279,  -0.3289,  -7.4438,   1.2123,   0.2799,   3.8951,  -0.7419,\n",
      "          2.9967,  -1.9475,   0.3669,   4.1705,   0.4131,  -0.8882,   0.9231,\n",
      "         -0.5112,  -4.2923,   4.6850,   4.2909,  -5.3442,   3.6022,   0.8800,\n",
      "         -3.3514,  -1.1483,   0.7188,  -1.4102,  -0.0244,   2.8213,  -0.3856,\n",
      "         -3.0411,  -3.1663,  -5.0189,  -3.6008,  -2.8663,   0.1998,   4.6103,\n",
      "         -4.7194,  -4.7101,  -0.3997,  -3.9943,   1.7160,  -0.0153,   3.2959,\n",
      "         -2.6514,   1.6684,   1.3652,  -1.5603,   0.4679,  -0.5590,  -1.0911,\n",
      "         -0.9756,  -0.3755], grad_fn=<SliceBackward0>)\n",
      "tensor([ 3.1955e+00,  5.4139e+00,  2.1630e+00,  1.0043e+00,  4.1027e+00,\n",
      "         3.6077e+00, -4.2609e+00,  2.6112e+00,  2.5508e+00,  1.8204e+00,\n",
      "        -3.5141e+00, -5.5689e+00, -2.3988e+00,  3.3773e-01,  2.5743e+00,\n",
      "         1.1659e-01, -9.1701e-01, -3.2852e+00, -2.5296e+00,  1.0206e-01,\n",
      "         1.6742e+00,  2.4063e-01,  4.1708e+00, -2.1607e+00,  1.8867e+00,\n",
      "         4.1153e+00,  1.7196e+00, -2.3008e+00, -6.5144e-01, -4.1186e+00,\n",
      "        -1.2046e+00,  3.2083e+00, -8.9915e-01,  3.7866e+00,  1.0251e+00,\n",
      "         1.0808e+00, -6.8857e-02,  6.8818e-01, -2.1025e+00,  3.6439e+00,\n",
      "         1.6686e-01, -3.5962e-01,  1.2836e+00, -1.3857e+00, -1.3637e-01,\n",
      "         8.2535e+00,  1.9286e+00,  3.7345e+00, -1.2021e+00,  2.1402e+00,\n",
      "         1.7624e-01, -3.6460e+00, -5.7081e-02, -3.7024e+00,  1.9605e+00,\n",
      "         1.0440e+01,  4.9133e+00, -2.4589e+00,  2.4076e+00,  8.1967e+00,\n",
      "        -6.2416e+00,  1.7704e+00,  2.2654e+00,  1.9994e+00, -4.3151e+00,\n",
      "        -2.0288e+00, -1.0601e-01, -5.5601e-01, -4.2540e-01,  2.6315e+00,\n",
      "         4.2551e+00,  6.0416e-01,  3.5534e-01,  2.9860e+00,  2.0260e+00,\n",
      "         3.8896e+00, -2.9529e-01, -9.9527e-01, -1.0691e+00,  4.3761e+00,\n",
      "         4.2178e+00,  2.5516e+00,  3.4706e-01, -2.7330e-03,  1.3537e+00,\n",
      "         3.9102e+00,  3.2443e-01,  7.5234e+00,  5.6582e-02, -7.4746e-01,\n",
      "        -2.9500e+00, -4.0204e+00, -4.0686e+00,  3.6389e+00,  1.9024e+00,\n",
      "         5.7815e-01,  7.3368e+00, -7.5310e+00,  8.6569e-01, -3.2475e+00,\n",
      "        -1.4035e+00, -2.1432e+00, -2.4589e+00,  4.6633e+00,  4.1190e+00,\n",
      "         4.4658e+00, -6.8875e-01, -1.3052e+00, -1.3965e+00,  6.6911e-01,\n",
      "        -5.8970e+00, -4.5051e+00,  4.9777e-01, -2.7483e+00, -1.0460e+00,\n",
      "        -9.4490e-01, -9.4460e-01,  2.4311e+00,  1.6387e+00,  4.8175e+00,\n",
      "         2.6257e+00, -1.4537e+00, -5.0766e+00,  1.8202e+00, -9.6604e-01,\n",
      "        -3.9444e+00, -3.2390e+00, -1.5274e+00], grad_fn=<SliceBackward0>)\n",
      "tensor([ 3.1955e+00,  5.4139e+00,  2.1630e+00,  1.0043e+00,  4.1027e+00,\n",
      "         3.6077e+00, -4.2609e+00,  2.6112e+00,  2.5508e+00,  1.8204e+00,\n",
      "        -3.5141e+00, -5.5689e+00, -2.3988e+00,  3.3773e-01,  2.5743e+00,\n",
      "         1.1659e-01, -9.1701e-01, -3.2852e+00, -2.5296e+00,  1.0206e-01,\n",
      "         1.6742e+00,  2.4063e-01,  4.1708e+00, -2.1607e+00,  1.8867e+00,\n",
      "         4.1153e+00,  1.7196e+00, -2.3008e+00, -6.5144e-01, -4.1186e+00,\n",
      "        -1.2046e+00,  3.2083e+00, -8.9915e-01,  3.7866e+00,  1.0251e+00,\n",
      "         1.0808e+00, -6.8858e-02,  6.8818e-01, -2.1025e+00,  3.6439e+00,\n",
      "         1.6686e-01, -3.5962e-01,  1.2836e+00, -1.3857e+00, -1.3637e-01,\n",
      "         8.2535e+00,  1.9286e+00,  3.7345e+00, -1.2021e+00,  2.1402e+00,\n",
      "         1.7624e-01, -3.6460e+00, -5.7081e-02, -3.7024e+00,  1.9605e+00,\n",
      "         1.0440e+01,  4.9133e+00, -2.4589e+00,  2.4076e+00,  8.1967e+00,\n",
      "        -6.2416e+00,  1.7704e+00,  2.2654e+00,  1.9994e+00, -4.3151e+00,\n",
      "        -2.0288e+00, -1.0601e-01, -5.5602e-01, -4.2541e-01,  2.6315e+00,\n",
      "         4.2551e+00,  6.0416e-01,  3.5534e-01,  2.9860e+00,  2.0260e+00,\n",
      "         3.8896e+00, -2.9529e-01, -9.9527e-01, -1.0691e+00,  4.3761e+00,\n",
      "         4.2178e+00,  2.5516e+00,  3.4706e-01, -2.7323e-03,  1.3537e+00,\n",
      "         3.9102e+00,  3.2443e-01,  7.5234e+00,  5.6582e-02, -7.4746e-01,\n",
      "        -2.9500e+00, -4.0204e+00, -4.0686e+00,  3.6389e+00,  1.9024e+00,\n",
      "         5.7815e-01,  7.3368e+00, -7.5310e+00,  8.6569e-01, -3.2475e+00,\n",
      "        -1.4035e+00, -2.1432e+00, -2.4589e+00,  4.6633e+00,  4.1190e+00,\n",
      "         4.4658e+00, -6.8875e-01, -1.3052e+00, -1.3965e+00,  6.6911e-01,\n",
      "        -5.8970e+00, -4.5051e+00,  4.9777e-01, -2.7483e+00, -1.0460e+00,\n",
      "        -9.4490e-01, -9.4460e-01,  2.4311e+00,  1.6387e+00,  4.8175e+00,\n",
      "         2.6257e+00, -1.4537e+00, -5.0766e+00,  1.8202e+00, -9.6604e-01,\n",
      "        -3.9444e+00, -3.2390e+00, -1.5274e+00], grad_fn=<SliceBackward0>)\n",
      "tensor([-4.4220,  1.0901, -1.6935,  4.3478, -1.3509,  0.5889,  3.8998, -4.8633,\n",
      "        -0.9690,  0.1896, -5.6392, -0.8945, -3.2611,  0.1713, -3.4014, -2.1330,\n",
      "        -4.0002, -0.3949, -5.5235, -0.1552, -3.8135, -0.3245, -2.4788, -3.0812,\n",
      "         2.9248, -0.7031, -2.8734,  0.5018, -1.4486, -0.3115,  2.9902,  1.4460,\n",
      "         1.5271,  0.2548,  4.0056, -0.0386, -3.7947, -3.9685, -5.6064, -2.6900,\n",
      "         5.1239, -2.4601,  0.6295,  2.0553, -0.5171, -3.0363,  3.1472,  2.6894,\n",
      "         3.5560,  3.2136,  2.4031, -1.5009, -1.6766, -1.7581, -1.5203,  1.4661,\n",
      "        -3.2267,  1.8556, -2.5085, -0.2572,  0.1147, -2.2295, -8.6341, -3.2425,\n",
      "        -4.1468, -0.9633, -2.6164, -1.3920, -1.5191,  2.0479,  1.5892, -0.2034,\n",
      "         2.8292,  0.7572,  0.8616,  1.4916, -0.3823, -3.9949, -5.9042, -1.7644,\n",
      "         1.5108,  3.0591,  1.9802,  2.3996, -1.9367,  6.6746,  1.4077,  2.7029,\n",
      "        -3.7618, -0.2771,  4.1056, -5.0029, -5.6002,  4.3783, -3.4791,  3.1581,\n",
      "         4.9613,  2.7924,  0.8123,  3.9128,  0.9868,  4.7895,  0.0741, -3.1290,\n",
      "        -5.5041, -0.6046, -1.8492,  1.7673,  3.4104,  0.6579,  1.8634, -1.3503,\n",
      "         2.4390, -0.9262, -5.3392,  0.5292, -4.8679,  1.5669, -1.8181, -2.3932,\n",
      "        -2.1447,  0.1445, -5.9608,  6.0297, -4.0142,  8.2683,  1.2363, -1.0973],\n",
      "       grad_fn=<SliceBackward0>)\n",
      "tensor([-4.4220,  1.0901, -1.6935,  4.3478, -1.3509,  0.5889,  3.8998, -4.8633,\n",
      "        -0.9690,  0.1896, -5.6392, -0.8945, -3.2611,  0.1713, -3.4014, -2.1330,\n",
      "        -4.0002, -0.3949, -5.5235, -0.1552, -3.8135, -0.3245, -2.4788, -3.0812,\n",
      "         2.9248, -0.7031, -2.8734,  0.5018, -1.4486, -0.3115,  2.9902,  1.4460,\n",
      "         1.5271,  0.2548,  4.0056, -0.0386, -3.7947, -3.9685, -5.6064, -2.6900,\n",
      "         5.1239, -2.4601,  0.6295,  2.0553, -0.5171, -3.0363,  3.1472,  2.6894,\n",
      "         3.5560,  3.2136,  2.4031, -1.5009, -1.6766, -1.7581, -1.5203,  1.4661,\n",
      "        -3.2267,  1.8556, -2.5085, -0.2572,  0.1147, -2.2295, -8.6341, -3.2425,\n",
      "        -4.1468, -0.9633, -2.6164, -1.3920, -1.5191,  2.0479,  1.5892, -0.2034,\n",
      "         2.8292,  0.7572,  0.8616,  1.4916, -0.3823, -3.9949, -5.9042, -1.7644,\n",
      "         1.5108,  3.0591,  1.9802,  2.3996, -1.9367,  6.6746,  1.4077,  2.7029,\n",
      "        -3.7618, -0.2771,  4.1056, -5.0029, -5.6002,  4.3783, -3.4791,  3.1581,\n",
      "         4.9613,  2.7924,  0.8123,  3.9128,  0.9868,  4.7895,  0.0741, -3.1290,\n",
      "        -5.5041, -0.6046, -1.8492,  1.7673,  3.4104,  0.6579,  1.8634, -1.3503,\n",
      "         2.4390, -0.9262, -5.3392,  0.5292, -4.8679,  1.5669, -1.8181, -2.3932,\n",
      "        -2.1447,  0.1445, -5.9608,  6.0297, -4.0142,  8.2683,  1.2363, -1.0973],\n",
      "       grad_fn=<SliceBackward0>)\n"
     ]
    }
   ],
   "source": [
    "#print(res[-1].sum(dim=(0,1))[:-3]-whole_res.sum(dim=(0,1))[:19])\n",
    "l_b, r_b = d_conv -1, -(d_conv-1)\n",
    "s_p = s\n",
    "print(res[0].sum(dim=(0,1))[0:r_b])\n",
    "print(whole_res.sum(dim=(0,1))[:s+d_conv-1])\n",
    "print(res[1].sum(dim=(0,1))[l_b:r_b])#[l_b:r_b])\n",
    "print(whole_res.sum(dim=(0,1))[s+d_conv-1:s*2+d_conv-1])\n",
    "print(res[2].sum(dim=(0,1))[l_b:r_b])\n",
    "print(whole_res.sum(dim=(0,1))[s*2+d_conv-1:s*3+d_conv-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "id": "0c5ad79f",
   "metadata": {},
   "outputs": [],
   "source": [
    "res[0] = res[0][:,:,:r_b]\n",
    "res[-1] = res[-1][:,:,l_b:]\n",
    "for i in range(1,len(res)-1):\n",
    "    res[i] = res[i][:,:,l_b:r_b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "id": "e2e48339-94e1-47a2-bf71-7c90c5bdcd37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.0000e+00,  0.0000e+00,  1.1269e-07,  ..., -1.9139e-07,\n",
       "         0.0000e+00,  0.0000e+00], grad_fn=<SumBackward1>)"
      ]
     },
     "execution_count": 486,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.cat(res,dim=-1)-whole_res).sum(dim=(0,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 487,
   "id": "39949265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-6.9360e-06, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 487,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(torch.cat(res,dim=-1)-whole_res).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "id": "15822b16",
   "metadata": {},
   "outputs": [],
   "source": [
    "seqlens = [170, 65, 100]\n",
    "\n",
    "xs = [torch.randint(0, 1000, (1, seqlen), device='cpu', dtype=torch.long) for seqlen in seqlens]\n",
    "\n",
    "    # Reference 1: Forward pass with seq_idx\n",
    "x = torch.cat(xs, dim=1)\n",
    "seq_idx = torch.cat([torch.full((ids.shape[1],), i, dtype=torch.int32, device='cpu')\n",
    "                         for i, ids in enumerate(xs)], dim=0).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "id": "63247392",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]],\n",
       "       dtype=torch.int32)"
      ]
     },
     "execution_count": 496,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 499,
   "id": "5932db10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
      ]
     },
     "execution_count": 499,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(range(10))[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd97b6e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mamba",
   "language": "python",
   "name": "mamba"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}