Preprocessing_MIDI_Dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Preprocessing MIDI Dataset",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kartikey-singh/Replicating-SeER/blob/master/Preprocessing_MIDI_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTbRpCB_nd9I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Switching to HIGH RAM Session, Only run when in 12GB Session\n",
        "a = []\n",
        "while(1):\n",
        "    a.append('1')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TEsUIJYw4aGm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import json\n",
        "import requests  \n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pickle\n",
        "from sklearn import preprocessing"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YDCaop4xWmL5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "5a27c17e-ece1-4ea7-be5b-81986c49cff8"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4gbF8NG50NI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a03e8e62-5248-4ead-f3c1-88768d863e92"
      },
      "source": [
        "PATH = r\"/content/drive/My Drive/dataset/\"\n",
        "try:\n",
        "  os.mkdir(PATH)\n",
        "except:\n",
        "  print(\"File already exists\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "File already exists\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DbLqCTxckXsI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file_urls = [\"http://millionsongdataset.com/sites/default/files/challenge/train_triplets.txt.zip\", \n",
        "        \"http://hog.ee.columbia.edu/craffel/lmd/lmd_matched.tar.gz\",\n",
        "        \"https://www.fourmilab.ch/webtools/midicsv/midicsv-1.1.tar.gz\",\n",
        "        \"http://millionsongdataset.com/sites/default/files/AdditionalFiles/unique_tracks.txt\",\n",
        "        \"http://hog.ee.columbia.edu/craffel/lmd/match_scores.json\"]\n",
        "\n",
        "file_names = [\"data.zip\", \n",
        "              \"data_midi.tar.gz\", \n",
        "              \"midicsv.tar.gz\", \n",
        "              \"unique_tracks.txt\",\n",
        "              \"match_scores.json\"]\n",
        "\n",
        "for file_name, file_url in zip(file_names, file_urls):              \n",
        "  r = requests.get(file_url, stream = True)  \n",
        "  \n",
        "  with open(PATH + file_name, \"wb\") as file:  \n",
        "      for block in r.iter_content(chunk_size = 1024): \n",
        "          if block:  \n",
        "              file.write(block)  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DcCa7SMS6PSk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ba09ce6f-8227-42cc-f2fd-99597d4248f7"
      },
      "source": [
        "# !unzip 'drive/My Drive/dataset/data.zip' -d 'drive/My Drive/dataset/'\n",
        "!tar -xf 'drive/My Drive/dataset/data_midi.tar.gz' --directory 'drive/My Drive/dataset/'\n",
        "!tar -xf 'drive/My Drive/dataset/midicsv.tar.gz' --directory 'drive/My Drive/dataset/'\n",
        "print(\"Done extracting datasets\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Done extracting datasets\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nWu7Qyrilqqj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233
        },
        "outputId": "21983c53-2f12-49a8-9402-0579011ca586"
      },
      "source": [
        "# Installing midicsv\n",
        "% cd drive/My\\ Drive/dataset/midicsv-1.1/\n",
        "! make\n",
        "! make check\n",
        "! make install\n",
        "% cd\n",
        "! ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/dataset/midicsv-1.1\n",
            "make: Nothing to be done for 'all'.\n",
            "make: execvp: ./midicsv: Permission denied\n",
            "Makefile:44: recipe for target 'check' failed\n",
            "make: *** [check] Error 127\n",
            "install -d -m 755 /usr/local/bin\n",
            "install -m 755 midicsv csvmidi /usr/local/bin\n",
            "install -d -m 755 /usr/local/man/man1\n",
            "install -m 644 midicsv.1 csvmidi.1 /usr/local/man/man1\n",
            "install -d -m 755 /usr/local/man/man5\n",
            "install -m 644 midicsv.5 /usr/local/man/man5\n",
            "/root\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cOEt3QU_crES",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 260
        },
        "outputId": "9922f28b-a8b4-4f98-a44d-8bb91246b0f8"
      },
      "source": [
        "track_song = pd.read_csv(PATH + 'unique_tracks.txt', sep=\"<SEP>\", header=None)\n",
        "track_song.rename(columns={0:'track', 1:'song', 2:'title', 3:'artist'}, inplace=True)\n",
        "track_song.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>track</th>\n",
              "      <th>song</th>\n",
              "      <th>title</th>\n",
              "      <th>artist</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TRMMMYQ128F932D901</td>\n",
              "      <td>SOQMMHC12AB0180CB8</td>\n",
              "      <td>Faster Pussy cat</td>\n",
              "      <td>Silent Night</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TRMMMKD128F425225D</td>\n",
              "      <td>SOVFVAK12A8C1350D9</td>\n",
              "      <td>Karkkiautomaatti</td>\n",
              "      <td>Tanssi vaan</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>TRMMMRX128F93187D9</td>\n",
              "      <td>SOGTUKN12AB017F4F1</td>\n",
              "      <td>Hudson Mohawke</td>\n",
              "      <td>No One Could Ever</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>TRMMMCH128F425532C</td>\n",
              "      <td>SOBNYVR12A8C13558C</td>\n",
              "      <td>Yerba Brava</td>\n",
              "      <td>Si Vos Querés</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>TRMMMWA128F426B589</td>\n",
              "      <td>SOHSBXH12A8C13B0DF</td>\n",
              "      <td>Der Mystic</td>\n",
              "      <td>Tangle Of Aspens</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                track                song             title             artist\n",
              "0  TRMMMYQ128F932D901  SOQMMHC12AB0180CB8  Faster Pussy cat       Silent Night\n",
              "1  TRMMMKD128F425225D  SOVFVAK12A8C1350D9  Karkkiautomaatti        Tanssi vaan\n",
              "2  TRMMMRX128F93187D9  SOGTUKN12AB017F4F1    Hudson Mohawke  No One Could Ever\n",
              "3  TRMMMCH128F425532C  SOBNYVR12A8C13558C       Yerba Brava      Si Vos Querés\n",
              "4  TRMMMWA128F426B589  SOHSBXH12A8C13B0DF        Der Mystic   Tangle Of Aspens"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVnjl7eRfX_w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "song_track = {}\n",
        "for song, track in zip(track_song['song'], track_song['track']):\n",
        "  song_track[song] = track"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3zV8i-Ko5xoE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "9606fa0e-b1c4-49aa-d47f-b22a9907874f"
      },
      "source": [
        "df = pd.read_csv(PATH + 'train_triplets.txt', sep='\\t', header=None)\n",
        "df.rename(columns={0:'user', 1:'song', 2:'play_count'}, inplace=True)\n",
        "df['track'] = df['song'].apply(lambda x: song_track[x])\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
              "      <td>SOAKIMP12A8C130995</td>\n",
              "      <td>1</td>\n",
              "      <td>TRIQAUQ128F42435AD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
              "      <td>SOAPDEY12A81C210A9</td>\n",
              "      <td>1</td>\n",
              "      <td>TRIRLYL128F42539D1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
              "      <td>SOBBMDR12A8C13253B</td>\n",
              "      <td>2</td>\n",
              "      <td>TRMHBXZ128F4238406</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
              "      <td>SOBFNSP12AF72A0E22</td>\n",
              "      <td>1</td>\n",
              "      <td>TRYQMNI128F147C1C7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
              "      <td>SOBFOVM12A58A7D494</td>\n",
              "      <td>1</td>\n",
              "      <td>TRAHZNE128F9341B86</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                       user  ...               track\n",
              "0  b80344d063b5ccb3212f76538f3d9e43d87dca9e  ...  TRIQAUQ128F42435AD\n",
              "1  b80344d063b5ccb3212f76538f3d9e43d87dca9e  ...  TRIRLYL128F42539D1\n",
              "2  b80344d063b5ccb3212f76538f3d9e43d87dca9e  ...  TRMHBXZ128F4238406\n",
              "3  b80344d063b5ccb3212f76538f3d9e43d87dca9e  ...  TRYQMNI128F147C1C7\n",
              "4  b80344d063b5ccb3212f76538f3d9e43d87dca9e  ...  TRAHZNE128F9341B86\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Be4Fxn6aexv3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Local path constants\n",
        "DATA_PATH = 'data'\n",
        "RESULTS_PATH = 'results'\n",
        "# Path to the file match_scores.json distributed with the LMD\n",
        "SCORE_FILE = os.path.join(RESULTS_PATH, 'match_scores.json')\n",
        "\n",
        "# Utility functions for retrieving paths\n",
        "def msd_id_to_dirs(msd_id):\n",
        "    \"\"\"Given an MSD ID, generate the path prefix.\n",
        "    E.g. TRABCD12345678 -> A/B/C/TRABCD12345678\"\"\"\n",
        "    return os.path.join(msd_id[2], msd_id[3], msd_id[4], msd_id)\n",
        "\n",
        "def msd_id_to_mp3(msd_id):\n",
        "    \"\"\"Given an MSD ID, return the path to the corresponding mp3\"\"\"\n",
        "    return os.path.join(DATA_PATH, 'msd', 'mp3',\n",
        "                        msd_id_to_dirs(msd_id) + '.mp3')\n",
        "\n",
        "def msd_id_to_h5(h5):\n",
        "    \"\"\"Given an MSD ID, return the path to the corresponding h5\"\"\"\n",
        "    return os.path.join(RESULTS_PATH, 'lmd_matched_h5',\n",
        "                        msd_id_to_dirs(msd_id) + '.h5')\n",
        "\n",
        "def get_midi_path(msd_id, midi_md5, kind):\n",
        "    \"\"\"Given an MSD ID and MIDI MD5, return path to a MIDI file.\n",
        "    kind should be one of 'matched' or 'aligned'. \"\"\"\n",
        "    return os.path.join(RESULTS_PATH, 'lmd_{}'.format(kind),\n",
        "                        msd_id_to_dirs(msd_id), midi_md5 + '.mid')\n",
        "    \n",
        "def path_exists(row):\n",
        "    if os.path.exists('/content/drive/My Drive/dataset/lmd_matched/' + msd_id_to_dirs(row['track'])):        \n",
        "      return True\n",
        "    return False      "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lnD4M44xe0-e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "SCORE_FILE = PATH + 'match_scores.json'\n",
        "\n",
        "with open(SCORE_FILE) as f:\n",
        "    scores = json.load(f)     "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j18ZRwr6fDce",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "track_exists = []\n",
        "for track in df['track']:\n",
        "  if track in scores.keys():\n",
        "    track_exists.append(True)\n",
        "  else:    \n",
        "    track_exists.append(False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MDAKzxKHjqhU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['midi_exists'] = track_exists\n",
        "df_ = df[df['midi_exists'] == True]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3PvxYnlxf9fd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "64fa24c9-2820-4216-b70b-67421770ab0b"
      },
      "source": [
        "print(df.shape, df_.shape)\n",
        "print(df['track'].unique().shape, df_['track'].unique().shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(384546,) (11890,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qvCI8GntiHxy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "a07cb1a3-5ea1-41ff-e1f9-0e1f58f3b028"
      },
      "source": [
        "# Group by users if less than 20 unique songs remove the user\n",
        "user_ = df_.groupby('user')['track'].nunique()\n",
        "mask_user = []\n",
        "\n",
        "for index, user in user_.items():\n",
        "  if user >= 28:\n",
        "    mask_user.append(index)  \n",
        "\n",
        "df1_ = df_[df['user'].isin(mask_user)]\n",
        "print(df1_.shape, df1_['track'].unique().shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(1331562, 5) (9545,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gwkFwfSn-pU9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4aa597c4-574f-4ca2-b28d-e8b1ff5d556e"
      },
      "source": [
        "# Removing Outliers\n",
        "df11_ = df1_[((df1_.play_count - df1_.play_count.mean()) / df1_.play_count.std()).abs() < 3]\n",
        "print(df11_.shape, df11_['track'].unique().shape, df11_['user'].unique().shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1318132, 5) (9529,) (32126,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KxF1GaiIlI_5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# df.to_csv(PATH + 'df.csv', index=False)\n",
        "# df_.to_csv(PATH + 'df_.csv', index=False)\n",
        "df11_.to_csv(PATH + 'df1_.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H7pi-60Z7_Rd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Validating All files exist in lmd_matched/\n",
        "count = 0\n",
        "for track in df1_['track'].unique():\n",
        "  if os.path.exists('/content/drive/My Drive/dataset/lmd_matched/' + msd_id_to_dirs(track)):\n",
        "    count = count + 1\n",
        "\n",
        "print(count, df1_['track'].unique().shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1YQ7KLBOXZP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# TODO\n",
        "# midicsv conversion\n",
        "# Median keeping"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wCIelubJBQQs",
        "colab_type": "text"
      },
      "source": [
        "# Stage 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XypjZPhsO9uR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "44e36e6b-f64b-4fd2-ab9a-76a6c190dfb6"
      },
      "source": [
        "df = pd.read_csv(PATH + 'df1_.csv')\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "      <th>midi_exists</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOANQFY12AB0183239</td>\n",
              "      <td>1</td>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOAYATB12A6701FD50</td>\n",
              "      <td>1</td>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBOAFP12A8C131F36</td>\n",
              "      <td>7</td>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBZZDU12A6310D8A3</td>\n",
              "      <td>7</td>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SODEAWL12AB0187032</td>\n",
              "      <td>8</td>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                       user  ... midi_exists\n",
              "0  d6589314c0a9bcbca4fee0c93b14bc402363afea  ...        True\n",
              "1  d6589314c0a9bcbca4fee0c93b14bc402363afea  ...        True\n",
              "2  d6589314c0a9bcbca4fee0c93b14bc402363afea  ...        True\n",
              "3  d6589314c0a9bcbca4fee0c93b14bc402363afea  ...        True\n",
              "4  d6589314c0a9bcbca4fee0c93b14bc402363afea  ...        True\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0m16wZB9n-6E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def paths(row, path):\n",
        "  midi_files = scores[row['track']]\n",
        "  max_similar_midi = max(midi_files, key=midi_files.get)\n",
        "  return path + msd_id_to_dirs(row['track']) + '/' + max_similar_midi + '.mid'\n",
        "\n",
        "# row = {'track': 'TRWMHMP128EF34293F'}\n",
        "# os.path.exists(paths(row, PATH + 'lmd_matched/'))  \n",
        "# PATH + 'lmd_matched/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_9JbiTPPpGG2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        },
        "outputId": "5bac4768-b7b5-4daf-86d4-2dd264e4b544"
      },
      "source": [
        "# Due to os.system requiring \\ between spaces in a name\n",
        "df['paths'] = df.apply(paths, axis=1, args={'/content/drive/My\\ Drive/dataset/lmd_matched/',})\n",
        "df.to_csv(PATH + 'df2_.csv', index=False)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "      <th>midi_exists</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOANQFY12AB0183239</td>\n",
              "      <td>1</td>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOAYATB12A6701FD50</td>\n",
              "      <td>1</td>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBOAFP12A8C131F36</td>\n",
              "      <td>7</td>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBZZDU12A6310D8A3</td>\n",
              "      <td>7</td>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SODEAWL12AB0187032</td>\n",
              "      <td>8</td>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                                              paths\n",
              "0           0  ...  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1           1  ...  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2           2  ...  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3           3  ...  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4           4  ...  /content/drive/My\\ Drive/dataset/lmd_matched/Q...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ZyxESHNsdwK",
        "colab_type": "text"
      },
      "source": [
        "# STAGE 1.1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4q49yncUr4KG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        },
        "outputId": "831f34c1-1938-4010-dad7-272e9349e41b"
      },
      "source": [
        "df = pd.read_csv(PATH + 'df2_.csv')\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "      <th>midi_exists</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOANQFY12AB0183239</td>\n",
              "      <td>1</td>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOAYATB12A6701FD50</td>\n",
              "      <td>1</td>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBOAFP12A8C131F36</td>\n",
              "      <td>7</td>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBZZDU12A6310D8A3</td>\n",
              "      <td>7</td>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SODEAWL12AB0187032</td>\n",
              "      <td>8</td>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                                              paths\n",
              "0           0  ...  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1           1  ...  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2           2  ...  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3           3  ...  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4           4  ...  /content/drive/My\\ Drive/dataset/lmd_matched/Q...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Qzoalq8BJdR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "outputId": "76d266fb-9ddf-4ad8-eb7d-062c6667c85b"
      },
      "source": [
        "df_ = df[['track', 'paths']]\n",
        "df_ = df_.drop_duplicates()\n",
        "print(df_.shape)\n",
        "df_.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(9529, 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>track</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                track                                              paths\n",
              "0  TROMKCG128F9320C09  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1  TREVCGZ128E079232A  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2  TRCPAGR128F423A01A  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3  TRGWWRE128E0781CDE  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4  TRQRVQG12903CAA7BD  /content/drive/My\\ Drive/dataset/lmd_matched/Q..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UXsCEdt2maR1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# os.system(\"mkdir -p /content/drive/My\\ Drive/dataset/temp\")\n",
        "nums = [x for x in range(1, 17)]\n",
        "tracks = {}\n",
        "def conversion():\n",
        "  midi_ = pd.read_csv('/content/temp.csv', header=None)\n",
        "  midi_.rename(columns={0: 'track', 1: 'time', 2: 'note_on_c', 3:'channel', 4:'note', 5:'velocity'}, inplace=True)\n",
        "  midi_.drop(columns=['track', 'note_on_c'], inplace=True)\n",
        "  midi_ = midi_.groupby(['time', 'channel']).mean()\n",
        "\n",
        "  midi_out = midi_.unstack().reindex(pd.MultiIndex.from_product([['note','velocity'], nums]), axis=1)\n",
        "  midi_out.columns = [f'C{j} {i}' for i,j in midi_out.columns]\n",
        "  midi_out = midi_out.fillna(0)\n",
        "  x = midi_out.values \n",
        "  return x.flatten()\n",
        "\n",
        "def one_at_a_time():\n",
        "  for index, row in df_.iterrows():     \n",
        "    os.system(f\"midicsv {row['paths']}| sed -E '/.*(Note_on)/!d' > /content/temp.csv\")    \n",
        "    tracks[row['track']] = conversion()    \n",
        "    yield row['track']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MwHy6yjbGKIt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395
        },
        "outputId": "42355ed8-2fc9-4254-8a22-6e9cde0d8bf2"
      },
      "source": [
        "count = 0\n",
        "for x in one_at_a_time():\n",
        "  if count > 20:\n",
        "    break\n",
        "  count = count + 1\n",
        "  print(x)\n",
        "  # print(track_id, tracks[track_id].shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TROMKCG128F9320C09\n",
            "TREVCGZ128E079232A\n",
            "TRCPAGR128F423A01A\n",
            "TRGWWRE128E0781CDE\n",
            "TRQRVQG12903CAA7BD\n",
            "TRVCUSW128F92F20C6\n",
            "TRZYJOO128F932FE77\n",
            "TRUATKW128F4227BCA\n",
            "TRLVQME128F931BAF3\n",
            "TRFHCVW128E078EB45\n",
            "TRCBRTN12903CC4BD1\n",
            "TRVSBTV12903CC6670\n",
            "TRXTTNI128F4295E0F\n",
            "TRONBHE128F92D6309\n",
            "TROAQBZ128F9326213\n",
            "TRMOYCC128C7196947\n",
            "TRWRGZD128E07857D2\n",
            "TROENBE128E0796854\n",
            "TROHFJK12903CC4BCE\n",
            "TRVTKOU128E078E0DA\n",
            "TROPUMP128F92EC162\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H__5pH19sXrC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "f = open(PATH + \"tracks_midi.pkl\", \"wb\")\n",
        "pickle.dump(tracks, f)\n",
        "f.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "luKJjK1prEV0",
        "colab_type": "text"
      },
      "source": [
        "# STAGE 1.2 SAMPLE DATA CREATION"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3okGRVXZpmhK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 344
        },
        "outputId": "96a35105-1ac6-4b1b-e054-75e3a0c2684c"
      },
      "source": [
        "# Check midicsv, scores, midi_to_dir cells have been executed\n",
        "df = pd.read_csv(PATH + 'df2_.csv')\n",
        "print(df.shape, df['user'].unique().shape, df['track'].unique().shape)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1318132, 7) (32126,) (9529,)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "      <th>midi_exists</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOANQFY12AB0183239</td>\n",
              "      <td>1</td>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOAYATB12A6701FD50</td>\n",
              "      <td>1</td>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBOAFP12A8C131F36</td>\n",
              "      <td>7</td>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBZZDU12A6310D8A3</td>\n",
              "      <td>7</td>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SODEAWL12AB0187032</td>\n",
              "      <td>8</td>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                                              paths\n",
              "0           0  ...  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1           1  ...  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2           2  ...  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3           3  ...  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4           4  ...  /content/drive/My\\ Drive/dataset/lmd_matched/Q...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oacp6qXspmst",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7eb62fbe-4297-46f9-8041-67c6db4a3d75"
      },
      "source": [
        "# User Wise Sampling\n",
        "tracks = list(df['track'].unique())[:1000]\n",
        "df_sample = df[df['track'].isin(tracks)]\n",
        "print(df_sample.shape, df_sample['user'].unique().shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(888373, 7) (32126,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G8xlo5k_pm2z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# User Wise Sampling\n",
        "# users = list(df['user'].unique())[:500]\n",
        "# df_sample = df[df['user'].isin(users)]\n",
        "# print(df_sample.shape, df_sample['track'].unique().shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y60s4Yi-rAsX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "outputId": "ad58e396-5c35-40e1-c040-7e4411bb9506"
      },
      "source": [
        "df_ = df_sample[['track', 'paths']]\n",
        "df_ = df_.drop_duplicates()\n",
        "print(df_.shape)\n",
        "df_.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000, 2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>track</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                track                                              paths\n",
              "0  TROMKCG128F9320C09  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1  TREVCGZ128E079232A  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2  TRCPAGR128F423A01A  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3  TRGWWRE128E0781CDE  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4  TRQRVQG12903CAA7BD  /content/drive/My\\ Drive/dataset/lmd_matched/Q..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3efv-FFJrA3s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# os.system(\"mkdir -p /content/drive/My\\ Drive/dataset/temp\")\n",
        "nums = [x for x in range(1, 17)]\n",
        "tracks = {}\n",
        "\n",
        "def conversion():\n",
        "  midi_ = pd.read_csv('/content/temp.csv', header=None)\n",
        "  midi_.rename(columns={0: 'track', 1: 'time', 2: 'note_on_c', 3:'channel', 4:'note', 5:'velocity'}, inplace=True)\n",
        "  midi_.drop(columns=['track', 'note_on_c'], inplace=True)\n",
        "  midi_ = midi_.groupby(['time', 'channel']).mean()\n",
        "\n",
        "  midi_out = midi_.unstack().reindex(pd.MultiIndex.from_product([['note','velocity'], nums]), axis=1)\n",
        "  midi_out.columns = [f'C{j} {i}' for i,j in midi_out.columns]\n",
        "  midi_out = midi_out.fillna(0)\n",
        "  x = midi_out.values \n",
        "  return x.flatten()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ne8YVmp6rBRX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "55003400-673e-460d-a620-63b442093cab"
      },
      "source": [
        "count = 0  \n",
        "for index, row in df_.iterrows():       \n",
        "  count = count + 1      \n",
        "  os.system(f\"midicsv {row['paths']}| sed -E '/.*(Note_on)/!d' > /content/temp.csv\")    \n",
        "  track_id = row['track']  \n",
        "  tracks[track_id] = conversion()      \n",
        "  print(count, track_id, tracks[track_id].shape)  \n",
        "\n",
        "with open(PATH + 'tracks_sample.pkl', 'wb') as fp:\n",
        "    pickle.dump(tracks, fp)  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1 TROMKCG128F9320C09 (65152,)\n",
            "2 TREVCGZ128E079232A (103680,)\n",
            "3 TRCPAGR128F423A01A (167424,)\n",
            "4 TRGWWRE128E0781CDE (13632,)\n",
            "5 TRQRVQG12903CAA7BD (33440,)\n",
            "6 TRVCUSW128F92F20C6 (182784,)\n",
            "7 TRZYJOO128F932FE77 (119392,)\n",
            "8 TRUATKW128F4227BCA (70432,)\n",
            "9 TRLVQME128F931BAF3 (26176,)\n",
            "10 TRFHCVW128E078EB45 (164000,)\n",
            "11 TRCBRTN12903CC4BD1 (138240,)\n",
            "12 TRVSBTV12903CC6670 (39936,)\n",
            "13 TRXTTNI128F4295E0F (210208,)\n",
            "14 TRONBHE128F92D6309 (26016,)\n",
            "15 TROAQBZ128F9326213 (148512,)\n",
            "16 TRMOYCC128C7196947 (69792,)\n",
            "17 TRWRGZD128E07857D2 (23072,)\n",
            "18 TROENBE128E0796854 (99520,)\n",
            "19 TROHFJK12903CC4BCE (38240,)\n",
            "20 TRVTKOU128E078E0DA (22496,)\n",
            "21 TROPUMP128F92EC162 (30528,)\n",
            "22 TRXWAZC128F9314B3E (50784,)\n",
            "23 TRSDWDN128F4274C03 (14784,)\n",
            "24 TRTEHXL128F931687B (31232,)\n",
            "25 TRWRXSJ128F9314B40 (30560,)\n",
            "26 TREDEFZ128F92F744F (254144,)\n",
            "27 TRNTALF128EF343800 (160576,)\n",
            "28 TRGAOLV128E0789D40 (52544,)\n",
            "29 TRENTGL128E0780C8E (43104,)\n",
            "30 TREPEEF128F9311356 (45408,)\n",
            "31 TRCFZKY128F425ACFE (23168,)\n",
            "32 TRPPVYA128F92FFF83 (108800,)\n",
            "33 TRZSLAK128F149F9B0 (269152,)\n",
            "34 TRKJBHI128F427A724 (82112,)\n",
            "35 TRTIMWD128F427CE31 (104768,)\n",
            "36 TRQFXKD128E0780CAE (54112,)\n",
            "37 TRNGAAK128F147DF92 (27328,)\n",
            "38 TRGRCFQ128F146DE19 (128896,)\n",
            "39 TRFWGOJ128E0780C8B (8640,)\n",
            "40 TRQUIHC128F42ADD0D (261248,)\n",
            "41 TRKFJXB128F42694A8 (128192,)\n",
            "42 TRYZXPW128F934B627 (182176,)\n",
            "43 TRGXCED128F92D1428 (139680,)\n",
            "44 TRTZNQZ12903CD044C (49216,)\n",
            "45 TRMVRJV128F1488FBC (16032,)\n",
            "46 TRHIVMU128F9327D14 (177728,)\n",
            "47 TRUOJJS128F145C256 (204672,)\n",
            "48 TRIMECG128F92D0147 (24832,)\n",
            "49 TRTQDGL128E0780C94 (39968,)\n",
            "50 TRTHGPK128F147DF90 (99776,)\n",
            "51 TRIDQJA12903CE29CC (45440,)\n",
            "52 TRWUPDA128E0780C8F (70176,)\n",
            "53 TRKGAYQ128F42827ED (7168,)\n",
            "54 TRNOHRW128F147DF91 (50464,)\n",
            "55 TRSZBPP128F42507C0 (275648,)\n",
            "56 TRYVBMA128E0789D39 (27264,)\n",
            "57 TRJCGAF128F92F1EDB (24768,)\n",
            "58 TRTZUYF128F92F57D8 (37248,)\n",
            "59 TRDGCIZ128F14640CE (141120,)\n",
            "60 TRGSPUQ128E0780DA5 (25344,)\n",
            "61 TRPRWYP128F146DE1D (160480,)\n",
            "62 TRCPXID128F92D5D3C (207200,)\n",
            "63 TRVUNHF128F9303DF4 (21120,)\n",
            "64 TRTXJBO128F4294115 (186176,)\n",
            "65 TRXWWWB128F9305CAA (317760,)\n",
            "66 TRARUTP128E0797FC7 (33472,)\n",
            "67 TRQUJJJ128F93450B9 (30080,)\n",
            "68 TRRJLQW128F4257946 (168640,)\n",
            "69 TRDGUPO128F9332CF6 (18880,)\n",
            "70 TRTIMHF128F426B41B (220864,)\n",
            "71 TRLQTMK128E07810A3 (132000,)\n",
            "72 TRCSJTV12903CD2DC4 (116256,)\n",
            "73 TROTSBD128E078ED75 (7936,)\n",
            "74 TRSSDHO128F9310EA5 (58528,)\n",
            "75 TRAURYF128F147805D (30336,)\n",
            "76 TRKDDZW128F930B393 (79328,)\n",
            "77 TRKLEJT128E078EC6C (10304,)\n",
            "78 TRJHWUC128F42AD812 (62496,)\n",
            "79 TRHTCXG12903CC2F60 (167168,)\n",
            "80 TRORPHN128E0797FC4 (64992,)\n",
            "81 TRHJRIH128F92FC6B7 (312512,)\n",
            "82 TRWAQOC12903CB84CA (108800,)\n",
            "83 TRTJCYB128F42BAC56 (29184,)\n",
            "84 TRYZBKZ128F1466CD6 (25792,)\n",
            "85 TRBKFKL128E078ED76 (13696,)\n",
            "86 TROTIUH128E0782538 (47904,)\n",
            "87 TRJCWTZ128E0797FCA (82752,)\n",
            "88 TRWMFMK128F424EA34 (118592,)\n",
            "89 TRRCPXP128E0791F01 (30432,)\n",
            "90 TRLAKFT128F427FF44 (73216,)\n",
            "91 TRXHCVJ128F4228C38 (133216,)\n",
            "92 TRMEBYQ128F932FEFF (229696,)\n",
            "93 TRGCHLH12903CB7352 (40032,)\n",
            "94 TRSSPPR128F92C3A4E (39392,)\n",
            "95 TRHLEVU128E0792E39 (392192,)\n",
            "96 TREBUCN128F92D2CAC (42464,)\n",
            "97 TRAALAH128E078234A (31264,)\n",
            "98 TRFSCIN128F92F4AD4 (203904,)\n",
            "99 TRMLBEG128F42BA124 (93280,)\n",
            "100 TRIZWXN128F423C6B4 (25440,)\n",
            "101 TREZILY128F934BF70 (13472,)\n",
            "102 TRIRCYZ128F423C331 (11712,)\n",
            "103 TRMXREA128F423C33A (12224,)\n",
            "104 TRWZIRS128F934BC0F (108832,)\n",
            "105 TRVLEST128E078BB53 (64480,)\n",
            "106 TRWUYHK128F1480FF9 (44288,)\n",
            "107 TRNWAPE128F934BC37 (156288,)\n",
            "108 TREUGGF128F1453A96 (26144,)\n",
            "109 TRCMLOO128F423C33D (75584,)\n",
            "110 TRJPSUJ128F423C32A (12640,)\n",
            "111 TRUZAOX128F9305498 (166816,)\n",
            "112 TRTFYJD128F1453A94 (14624,)\n",
            "113 TRUEXUA128F1453AC4 (29536,)\n",
            "114 TRGHMHI128F9332C43 (48832,)\n",
            "115 TROEEIY12903CCF25A (121792,)\n",
            "116 TRAJGDN128F1453AC3 (8768,)\n",
            "117 TRBSDYE128E078BB58 (19424,)\n",
            "118 TREWPDP128E07896EE (232224,)\n",
            "119 TRLTMTJ128F1453ABE (21376,)\n",
            "120 TRJYINF128F146B5F1 (210336,)\n",
            "121 TRIITXF128F147EFD5 (57472,)\n",
            "122 TRJLORC128F429596A (18912,)\n",
            "123 TRFWJHA128EF3425DB (105152,)\n",
            "124 TRAVMND128F4275780 (55264,)\n",
            "125 TRBZHPO128F147EFD3 (27296,)\n",
            "126 TRMVHYS128F423C6A8 (10880,)\n",
            "127 TRWEKDR128F9320BC6 (41600,)\n",
            "128 TRZLPJO128F423C693 (33344,)\n",
            "129 TRWMLQV12903CEA4FD (252064,)\n",
            "130 TRFDSKQ128F423C31F (37312,)\n",
            "131 TRGCMVF128F423C6AB (120768,)\n",
            "132 TRWDUUK128F1453ABD (41504,)\n",
            "133 TRFEEKR128F92EFEF6 (68384,)\n",
            "134 TRXXCIZ128F42574DD (208000,)\n",
            "135 TRFUPBA128F934F7E1 (197312,)\n",
            "136 TRVPMAE128F9334013 (128672,)\n",
            "137 TRTFRMJ128F93043F9 (59104,)\n",
            "138 TRVRIEM128F92EB8F6 (11840,)\n",
            "139 TRCGAVX12903CFBC61 (14656,)\n",
            "140 TRFGQVP128F93321F7 (26784,)\n",
            "141 TRRRYBV128E0793857 (35776,)\n",
            "142 TRWCIAX128F42925BD (170432,)\n",
            "143 TRFQTEX128E078E92D (122048,)\n",
            "144 TRGSVLV128F1493329 (155680,)\n",
            "145 TROAXUL128F422EC49 (91168,)\n",
            "146 TRJIZEK128F42AC2A0 (38464,)\n",
            "147 TRDKELF128F93210CA (52352,)\n",
            "148 TRDXOZM128E0793855 (16832,)\n",
            "149 TRQAHUW128F42933D1 (27360,)\n",
            "150 TRFYOIE128F93207CB (31296,)\n",
            "151 TRPYYZN128F9304FBA (92960,)\n",
            "152 TRYAJZI128F42727D5 (51360,)\n",
            "153 TRNEWWX128F9336A1F (137280,)\n",
            "154 TRXMRWR128F934BE2F (199808,)\n",
            "155 TRTCLFS128E0799AD4 (61952,)\n",
            "156 TRMDHCG128F4262B8C (7552,)\n",
            "157 TRKCZIZ128F92FD7DA (27008,)\n",
            "158 TRONMTE128F1458B51 (124800,)\n",
            "159 TRXXCTX128F9334C13 (177728,)\n",
            "160 TRPSSCO128F426AF1F (212448,)\n",
            "161 TRHQQGM128E0781C67 (38944,)\n",
            "162 TRVZTWF128F14A2B5D (224672,)\n",
            "163 TRZVPZQ128E078F0E3 (138112,)\n",
            "164 TRIGXQN128E078FEC7 (110560,)\n",
            "165 TRLFTLE128F425AFBD (23840,)\n",
            "166 TRBDSWA128F92FE70A (108000,)\n",
            "167 TRGAVTO12903CF4801 (86432,)\n",
            "168 TRUXXQM128E078407D (144288,)\n",
            "169 TRPTWLU128E078584F (56160,)\n",
            "170 TRTSQLA12903D0737C (41024,)\n",
            "171 TRQAGHX128F425EEBD (132544,)\n",
            "172 TRNMPMJ128F92FD7D2 (131840,)\n",
            "173 TRZHYJX128F92EB952 (391104,)\n",
            "174 TRMFSYX12903CAAA05 (47840,)\n",
            "175 TRDLVAD128F42B6138 (31360,)\n",
            "176 TROQXXB128F14546DA (28640,)\n",
            "177 TRTWFKE128EF35F93B (30528,)\n",
            "178 TRIWQUF128F14A2B5B (102144,)\n",
            "179 TRHDJDB128F930527A (166080,)\n",
            "180 TRLEPFM128F147FCB0 (104608,)\n",
            "181 TRBAUVN128F932FEF8 (140352,)\n",
            "182 TRRUXAH128E078FECB (65056,)\n",
            "183 TRANKTK128E07921D9 (49088,)\n",
            "184 TRKUOQD12903CE5A2B (51168,)\n",
            "185 TRHPOMN128F145247C (29152,)\n",
            "186 TRWVXJG128F93376A0 (70720,)\n",
            "187 TRGGNQM12903CBF862 (121760,)\n",
            "188 TRTVTFT128F9327735 (85088,)\n",
            "189 TREDKPE128F4279F9F (23584,)\n",
            "190 TRFUJBZ128F1485E14 (163520,)\n",
            "191 TRBNCXK128F147CBC6 (164448,)\n",
            "192 TRVCBXA128E0785856 (23360,)\n",
            "193 TRMQRTH128F42592D3 (46912,)\n",
            "194 TRUPWZN128E0785850 (29536,)\n",
            "195 TRNSIFK12903CC39E0 (160448,)\n",
            "196 TRQJIXA128F428DFCF (152064,)\n",
            "197 TRMJSKC12903CDA4CD (224064,)\n",
            "198 TRFSFEB12903CA144F (112704,)\n",
            "199 TRFDCPI128F93234B7 (28608,)\n",
            "200 TRONVVO128F92FD7D7 (284000,)\n",
            "201 TRVVTWQ128E078F1B7 (136192,)\n",
            "202 TRDCOWK128F92C5047 (88992,)\n",
            "203 TRFXKPH128E0793B8E (286240,)\n",
            "204 TRDQDOY128F147FC9E (55968,)\n",
            "205 TRUNSXQ128E078B97D (65344,)\n",
            "206 TRXBKFN128E078B97B (324768,)\n",
            "207 TRAAGCZ128F93210FD (214784,)\n",
            "208 TRXLZRG128F42836AC (155744,)\n",
            "209 TRYIFWK128E078B971 (119328,)\n",
            "210 TRVSOPA128F426B3EB (82176,)\n",
            "211 TRQQVBA12903CF9FEA (151136,)\n",
            "212 TRQQRTT128F92EFFF5 (105760,)\n",
            "213 TRKIZQR128F42289E1 (28192,)\n",
            "214 TRHASWP128E078E8D6 (221440,)\n",
            "215 TRALLSG128F425A685 (75808,)\n",
            "216 TRZXAGH128F93232BE (17600,)\n",
            "217 TRXQDIQ128E078868E (71104,)\n",
            "218 TRIGNCN12903CA9E3C (280896,)\n",
            "219 TRUDNRB128F42598CA (72768,)\n",
            "220 TRNXTXH128F92FF599 (176224,)\n",
            "221 TRGAJXT128F147848E (25792,)\n",
            "222 TRZYGSG128F9310CDB (81344,)\n",
            "223 TRYUXQW128E078B982 (89728,)\n",
            "224 TRADFCO12903CA9439 (81760,)\n",
            "225 TRFXQCT128EF35A24A (71104,)\n",
            "226 TRLVJCN128F92CBE35 (184320,)\n",
            "227 TRNBMUG128F92CB2B2 (195904,)\n",
            "228 TRPTROP128F427C0A9 (27840,)\n",
            "229 TRGHGIV128F42744C9 (211424,)\n",
            "230 TRMQSZG12903CA04D5 (64672,)\n",
            "231 TRYYMYX12903CA7DFA (44576,)\n",
            "232 TRFGZPB128F92FFBF9 (63360,)\n",
            "233 TRLVJOB128E078B97F (67712,)\n",
            "234 TRFTNPE128E078868D (125952,)\n",
            "235 TROCLAV128E0785A42 (153728,)\n",
            "236 TRDGZHV128F42AC320 (55072,)\n",
            "237 TROIYKU128F42884D9 (159616,)\n",
            "238 TRPFTWB128E078B974 (54880,)\n",
            "239 TRZOAFP128F9313EFF (171520,)\n",
            "240 TRFIUKA128F42644AA (132928,)\n",
            "241 TRFXAVZ128F146E04B (104032,)\n",
            "242 TRONJWZ128F428E424 (139104,)\n",
            "243 TREUBKI128F42597BE (192992,)\n",
            "244 TRRBEIE128E0782B31 (259744,)\n",
            "245 TRJWXTN128F42487F0 (287424,)\n",
            "246 TRVMMBF128F145B2EB (209568,)\n",
            "247 TRHRMNF128F14595A9 (155648,)\n",
            "248 TRDYZPO128F92C3BB1 (44032,)\n",
            "249 TRYXSYH128F93510D2 (325216,)\n",
            "250 TRDJRDU128E07914D3 (245376,)\n",
            "251 TRZTTWD128F149158B (29792,)\n",
            "252 TRNJGTR128F145958E (82784,)\n",
            "253 TRAPHVG128F14595A7 (119680,)\n",
            "254 TRZAQAZ128F149F9FF (142336,)\n",
            "255 TROIESH128F422EEA3 (266112,)\n",
            "256 TRXUYKI128F145288D (33184,)\n",
            "257 TRUYIOW12903CD6F99 (7584,)\n",
            "258 TRNQGGL128F4214A9E (44960,)\n",
            "259 TREHPEC128F92C3BB8 (222240,)\n",
            "260 TRYTOLM128F145B14D (60864,)\n",
            "261 TRADZXK128F429DB92 (143168,)\n",
            "262 TRSGUYX128F14667D9 (96064,)\n",
            "263 TRKWDZM128F4282621 (116320,)\n",
            "264 TRJHIKI128F42539D8 (268704,)\n",
            "265 TRBXZUT128F92D2C60 (30080,)\n",
            "266 TRTIRFG128F428D2F8 (109440,)\n",
            "267 TRNWQGU128F9304A23 (261088,)\n",
            "268 TRMZLZX128F146E04D (265888,)\n",
            "269 TRYQBGS128F92F23B1 (23040,)\n",
            "270 TRZVSIO128F428E536 (86208,)\n",
            "271 TRRQXOZ128E07860F8 (45600,)\n",
            "272 TRKFBKX128F4292AE1 (28896,)\n",
            "273 TRKJTUV128F932BB1D (44544,)\n",
            "274 TRKFULA128F4292AE4 (41152,)\n",
            "275 TRNVGHA128E07912F3 (68256,)\n",
            "276 TROUSOR128F92D3C14 (38272,)\n",
            "277 TRSFIBH128F42A9A93 (89632,)\n",
            "278 TRMNQZI128F42B58E8 (89088,)\n",
            "279 TRLWCQR128F14824EB (30432,)\n",
            "280 TRYNFQG128F427C721 (8192,)\n",
            "281 TRQWGHU128E07912FF (24800,)\n",
            "282 TRRHTLO128F9333BCE (194912,)\n",
            "283 TRGFYCL128F4243320 (334112,)\n",
            "284 TRMBZVF128F428507C (215552,)\n",
            "285 TRBHKVD128F146F758 (155520,)\n",
            "286 TRIIFIA128E07912F6 (68000,)\n",
            "287 TRKLATD128F9338236 (67680,)\n",
            "288 TRIKCQI128F92DDC16 (61152,)\n",
            "289 TREZZSP128F149EAFF (168800,)\n",
            "290 TRXYLKX128F148988C (103360,)\n",
            "291 TRYHOJR12903CAF098 (160480,)\n",
            "292 TRPDQTF128F92D3C11 (59584,)\n",
            "293 TRXDYUE128F429FC3F (46784,)\n",
            "294 TRBMNAU128F42B58E9 (73312,)\n",
            "295 TRIGUOP128F422B8B6 (96480,)\n",
            "296 TRADCIF128F9338278 (52224,)\n",
            "297 TRFWIRB128F42B6554 (72480,)\n",
            "298 TRGVSMR128F42B58E7 (73024,)\n",
            "299 TRSOHMN128F429388A (53632,)\n",
            "300 TRDPYXF128F933FD80 (111296,)\n",
            "301 TRAQJMP128F92E3821 (90816,)\n",
            "302 TRIEAAR12903CB78F4 (38720,)\n",
            "303 TROAZVW128F4279319 (86496,)\n",
            "304 TRPNFHR128F42430D9 (23584,)\n",
            "305 TRKEHNP128F145E1A5 (71104,)\n",
            "306 TRCTSAR128F9306235 (209216,)\n",
            "307 TRZBHWT128F42963D2 (44608,)\n",
            "308 TRAEHHJ12903CF492F (121728,)\n",
            "309 TRQWKDK128F426A901 (122016,)\n",
            "310 TRLMFRB128E078E1FA (159264,)\n",
            "311 TRBQIKR128F148BE55 (102240,)\n",
            "312 TRWYDVA128F92FDD86 (183520,)\n",
            "313 TRKQKWL128F92ED322 (26880,)\n",
            "314 TRLGMFJ128F4217DBE (15680,)\n",
            "315 TRILORM128F424315E (50080,)\n",
            "316 TRKZYBB128F92CC97D (81984,)\n",
            "317 TROWQGL128E0784BAA (45376,)\n",
            "318 TRMZUJS128F42957DC (114848,)\n",
            "319 TRJGQCQ128E078DFBA (129312,)\n",
            "320 TRNQDPE128F93164FF (62528,)\n",
            "321 TRTCMKQ128E079654D (161696,)\n",
            "322 TREUBTU12903CC84A6 (7264,)\n",
            "323 TRGBGVQ128F92CEAA2 (254368,)\n",
            "324 TRNFXMN128F92FC655 (165152,)\n",
            "325 TRHROOI128F428BA16 (105280,)\n",
            "326 TRHUACP128F9330D0C (265824,)\n",
            "327 TRYZUSO128F42BA4E2 (156736,)\n",
            "328 TRXXJTC128E078B306 (25056,)\n",
            "329 TRAJWBT128F932DEBE (42496,)\n",
            "330 TRKPTQJ128E078B065 (161376,)\n",
            "331 TRJZMIE128E0794201 (94112,)\n",
            "332 TRDJQLI128E07917C7 (32672,)\n",
            "333 TRWPUUA128F933582B (53408,)\n",
            "334 TRTWLRH128F426264F (132320,)\n",
            "335 TROTLOB128F1453462 (30816,)\n",
            "336 TRKZTAU128F42137C7 (98752,)\n",
            "337 TRMCTYQ128E07921DC (298272,)\n",
            "338 TRPJUXK128F9322C71 (30848,)\n",
            "339 TRFSEYG128F42BCE7E (110592,)\n",
            "340 TRTEEXF128F92E26B6 (31264,)\n",
            "341 TRUJYEJ128F92CEAA9 (174560,)\n",
            "342 TRTGUWJ128F146EC54 (84352,)\n",
            "343 TRXMJMD128F146AE5B (147712,)\n",
            "344 TRNXKLQ128E078B313 (31328,)\n",
            "345 TRCZGBG128F4222F61 (9280,)\n",
            "346 TRBXAHV128F9316893 (66816,)\n",
            "347 TRJJHFE128EF3518FD (58016,)\n",
            "348 TRXGVYV128F1463447 (11456,)\n",
            "349 TRHVBZP128F933ADCF (201408,)\n",
            "350 TRXUTLD128F930F92E (97472,)\n",
            "351 TRMCNML128EF35191B (125344,)\n",
            "352 TRKSUEH128E0787846 (41056,)\n",
            "353 TRNIEVN128F148D021 (282752,)\n",
            "354 TRTSLWB128F42675C6 (181056,)\n",
            "355 TRSYEAJ128F933A074 (27040,)\n",
            "356 TRQKFUN128F148D01E (93312,)\n",
            "357 TRRBOFU128EF3518E1 (37856,)\n",
            "358 TRMVHUY128E0791209 (104896,)\n",
            "359 TRONCBQ128EF3518FF (27776,)\n",
            "360 TRQAPRM128F9310CCE (121504,)\n",
            "361 TRAYVWN128E079120C (39968,)\n",
            "362 TRFYUEZ128F9310CD4 (87136,)\n",
            "363 TRGXZRI128F148D017 (223520,)\n",
            "364 TRQLTTH128E0791212 (34784,)\n",
            "365 TRDNDNY128F14A2F40 (21056,)\n",
            "366 TRPWMJJ128E079120E (26912,)\n",
            "367 TRYCCQR12903CF16B1 (18464,)\n",
            "368 TRZQKDS128F92F22F3 (65536,)\n",
            "369 TRHDZZE128E0783BE4 (131712,)\n",
            "370 TRHVZYA128E07848F1 (80192,)\n",
            "371 TRWPQNY128F92DF9BF (194624,)\n",
            "372 TRSDDGM128F14A421D (151616,)\n",
            "373 TRXZKGS128F429E4EE (204800,)\n",
            "374 TRDGQBZ128F4268240 (222240,)\n",
            "375 TRGVYIX128F4291DB3 (75552,)\n",
            "376 TRDZKEQ128F92D3C3C (217152,)\n",
            "377 TRLPNCW128F1483681 (273152,)\n",
            "378 TROFAON128E079371E (46112,)\n",
            "379 TROUHQM128F92C5777 (36160,)\n",
            "380 TROXMUA128F1491A41 (22976,)\n",
            "381 TROZAQQ12903CC7E4C (57216,)\n",
            "382 TRQBDQA128E079235D (145568,)\n",
            "383 TRQQOKT128E078C3AD (422912,)\n",
            "384 TROAISE128F933DF0C (217568,)\n",
            "385 TRNNBXN128F1459E1E (65024,)\n",
            "386 TRZCSRP128F93389F6 (258848,)\n",
            "387 TRKSVNB128F427D53A (378848,)\n",
            "388 TRBHLXQ128E07856AC (92192,)\n",
            "389 TRYXNOB128F934B60B (69248,)\n",
            "390 TRVYICQ128F4252493 (169952,)\n",
            "391 TRQGRXB128E078AA08 (177056,)\n",
            "392 TREHRDF128F426B05E (96224,)\n",
            "393 TRQAUIE128F14531EB (197440,)\n",
            "394 TRPBFWT128F92F7F1E (215264,)\n",
            "395 TREWPTU128F4233620 (265376,)\n",
            "396 TRQFHAC128F429411B (70112,)\n",
            "397 TRJFAOJ128F931D308 (97280,)\n",
            "398 TRVGWTX128F92E6F9A (43136,)\n",
            "399 TRLABJZ128F42A0DD7 (266912,)\n",
            "400 TRPGVYK128F42582D3 (169440,)\n",
            "401 TRPDNSO128F425EA35 (51904,)\n",
            "402 TRGBFCA128F423FC5D (45120,)\n",
            "403 TRKQWMI128E0795218 (16128,)\n",
            "404 TRHNGRY128E0791670 (159872,)\n",
            "405 TRIVMCG128F932105E (34560,)\n",
            "406 TRBXOCM128F92EF9C4 (225312,)\n",
            "407 TRPIUDL128F422890D (27264,)\n",
            "408 TRILDNT128F1451F15 (12864,)\n",
            "409 TRJOTOI128F14AAEC4 (38208,)\n",
            "410 TRKPWGR128E078EE06 (27808,)\n",
            "411 TREHPDX128F92DC86A (6240,)\n",
            "412 TRKRHYM128F42934A9 (28128,)\n",
            "413 TRIFVDW128F4279DE0 (44128,)\n",
            "414 TRLWRJX128F4268B74 (143616,)\n",
            "415 TRIEYZX12903CBCC61 (43264,)\n",
            "416 TRPDJBX128F42ACF9B (43648,)\n",
            "417 TRSQFGJ128F93059EF (44480,)\n",
            "418 TRKRZLR128F427255A (26816,)\n",
            "419 TREIMYW128F93366A2 (53024,)\n",
            "420 TREAUJQ128F147CC73 (54656,)\n",
            "421 TRSQBCK12903D080B2 (39360,)\n",
            "422 TRFEVKY128F4289414 (36160,)\n",
            "423 TRMQIPU128F4289418 (38368,)\n",
            "424 TRLQEIK128F4240959 (150496,)\n",
            "425 TRDBQXT128F4289423 (23872,)\n",
            "426 TRHJJZE128F14A2B3D (93440,)\n",
            "427 TROHLIV12903CBCBE6 (77824,)\n",
            "428 TRHCCXP128E0796554 (138688,)\n",
            "429 TRHBLDA12903CC32E9 (20960,)\n",
            "430 TRBBAEQ128F42B18B9 (48192,)\n",
            "431 TREQPIV12903CE0B7E (61984,)\n",
            "432 TRQSJOJ128F931BD15 (61984,)\n",
            "433 TRKZOBJ128F428AF61 (31008,)\n",
            "434 TRYPJVB128F93210C9 (162688,)\n",
            "435 TRHEOIY128E079654E (126080,)\n",
            "436 TRTTEOB128F426A3F0 (272096,)\n",
            "437 TRWXHVD128F42BAC6B (109984,)\n",
            "438 TRAQDUU128F92F036C (270688,)\n",
            "439 TRDGUAJ128F92D541D (10048,)\n",
            "440 TRKVYCJ12903CFE512 (50752,)\n",
            "441 TRELQBX128F146DE1F (23168,)\n",
            "442 TRWVOEA12903D09F58 (92064,)\n",
            "443 TRLOMKL128F4295BDA (27552,)\n",
            "444 TRQBRMK128F425D64B (39456,)\n",
            "445 TRIMFGK128F42BAC92 (20576,)\n",
            "446 TROLJKM128E07911F2 (119232,)\n",
            "447 TRNXWKG128E07927AC (110016,)\n",
            "448 TRHHZLB128F93293E4 (39296,)\n",
            "449 TRCGKRC128E0781A0E (159872,)\n",
            "450 TRMAKPC128F4272A45 (108032,)\n",
            "451 TRPKPTG128E07869D0 (114688,)\n",
            "452 TRVYTBA12903CAB304 (157952,)\n",
            "453 TRKROJB128F42725F8 (106176,)\n",
            "454 TROPNWM128F1488325 (142176,)\n",
            "455 TRWOMMO128E07944B2 (53664,)\n",
            "456 TRECRGP128F1470AC7 (76960,)\n",
            "457 TRXPEQW128F92E92CF (26048,)\n",
            "458 TRLAWLZ128E07890D4 (102624,)\n",
            "459 TRPUCNU128E0786193 (60768,)\n",
            "460 TREKDZI128F146A972 (95232,)\n",
            "461 TRTDRIP128F1470865 (185408,)\n",
            "462 TRKQJAZ128F147D45E (192256,)\n",
            "463 TRGRWLW128F14A8B61 (320608,)\n",
            "464 TRJIXGC128E0798B97 (159744,)\n",
            "465 TRNXNBL128F146E4DE (118496,)\n",
            "466 TRKQNZA128E0782CD6 (53824,)\n",
            "467 TRAXQYP128F930458D (76512,)\n",
            "468 TRPGXJR128F4278388 (111712,)\n",
            "469 TRCMAHC128F424E83E (148352,)\n",
            "470 TRNMKEH128F42B62AD (154688,)\n",
            "471 TRRQQYY128F933D800 (18080,)\n",
            "472 TRSMLZI12903CC830E (90720,)\n",
            "473 TRWYVBA128F424D7FC (50944,)\n",
            "474 TREZDXZ128E0780C30 (96896,)\n",
            "475 TRIMGYM128E07848C0 (37984,)\n",
            "476 TRGQDXC128F428A179 (73440,)\n",
            "477 TRFKGJG128F1497C8D (128672,)\n",
            "478 TRLHJXQ128E0781AE6 (149216,)\n",
            "479 TRVIOPH128E079894E (80480,)\n",
            "480 TRSIBNI12903CC833B (51040,)\n",
            "481 TRVCNDL128F9320C3A (67776,)\n",
            "482 TRNDZJT128E0792721 (60736,)\n",
            "483 TRGZDAD128E078F2EC (128224,)\n",
            "484 TRYSRXA128E079141B (224480,)\n",
            "485 TRTYYTO12903CA4A34 (258848,)\n",
            "486 TRALMAU128F4252947 (205792,)\n",
            "487 TRZZWIK128F428CFF2 (100224,)\n",
            "488 TREWFGS128F93075E5 (38048,)\n",
            "489 TRODJJP12903CB0D5C (248800,)\n",
            "490 TROTIXO128F427EA49 (211488,)\n",
            "491 TRNCOSA128F4233DD7 (134944,)\n",
            "492 TRGAGTH12903C994AE (82688,)\n",
            "493 TRJUIAR128F424169C (122240,)\n",
            "494 TRKJRVK128F932A6C9 (42048,)\n",
            "495 TRCCMHE128F429530D (44576,)\n",
            "496 TRMKCDS128F424315F (38784,)\n",
            "497 TRMVKSL128F14640E0 (160192,)\n",
            "498 TRLAWLV128F934B7B8 (67424,)\n",
            "499 TRWGTRJ128F42756E4 (298400,)\n",
            "500 TRYRIGS12903CEF7A2 (39584,)\n",
            "501 TRVRDYC128F1492397 (260992,)\n",
            "502 TRCUJKA128E07927AE (189216,)\n",
            "503 TREREBE128E078EB58 (19520,)\n",
            "504 TRCZTPL128F14920CD (71456,)\n",
            "505 TRXAQAU128F427FAE6 (70240,)\n",
            "506 TRDEGGZ128F1498BDA (28192,)\n",
            "507 TRXIOUQ128F4269099 (47776,)\n",
            "508 TRXZBFS128F92F24D2 (63776,)\n",
            "509 TRXRZEJ128F4274A89 (21024,)\n",
            "510 TRNICFB128F14773EB (108032,)\n",
            "511 TRFWGOY128F148B015 (147040,)\n",
            "512 TRUKYAT128F9301247 (29568,)\n",
            "513 TRAWOPG128F92EF9A1 (132000,)\n",
            "514 TRYDPVE128F426A8FA (107808,)\n",
            "515 TRASQGR128F14557AA (93280,)\n",
            "516 TRKDVCV128E0791712 (152576,)\n",
            "517 TRZSBTE12903CD7F93 (152128,)\n",
            "518 TRYSFAY12903CECB7D (159520,)\n",
            "519 TRUXLWW128F14567C4 (93248,)\n",
            "520 TRCOXGB128F42640E9 (28096,)\n",
            "521 TRKRKTZ128F14A82F5 (190944,)\n",
            "522 TRATKVY128EF35F97A (148800,)\n",
            "523 TRBBYVD128F931AE0C (72256,)\n",
            "524 TRHDFYF128F1459959 (96160,)\n",
            "525 TRTIYLR128F427B697 (100576,)\n",
            "526 TRGIOYX128F426A837 (239680,)\n",
            "527 TRRZDAU128F1472584 (71552,)\n",
            "528 TRRVASR128F14567C5 (44640,)\n",
            "529 TRZZEHL128EF341F32 (76352,)\n",
            "530 TRPCKQK128F424E12C (179488,)\n",
            "531 TRHDPWS128E0792432 (198144,)\n",
            "532 TRFLNKX128F9339775 (78432,)\n",
            "533 TRTKZCA128F1479BEE (269536,)\n",
            "534 TRBIXDZ128F427E461 (51840,)\n",
            "535 TRYEONX128F92F014F (46848,)\n",
            "536 TRFZDAB128F428CAA5 (26976,)\n",
            "537 TRBIBLJ12903CCF436 (134784,)\n",
            "538 TRVVWFP12903CAA89C (31616,)\n",
            "539 TRZJYXG128E078E9C4 (26944,)\n",
            "540 TRDAWVS128F4236D29 (226816,)\n",
            "541 TRZDPAF128F92F7C1F (12064,)\n",
            "542 TRZRJCI128F92F7C21 (6688,)\n",
            "543 TRZMEMN128F149CB7B (32800,)\n",
            "544 TRSBBOA128F145B794 (110656,)\n",
            "545 TRWDPTD12903D018B5 (19712,)\n",
            "546 TRGGSUX128F4282367 (121952,)\n",
            "547 TRPOWLW128E07930BB (48032,)\n",
            "548 TRUEUDK128E0782EA2 (32800,)\n",
            "549 TRBLHNN128E0792486 (211136,)\n",
            "550 TRQFZEO128E07930BC (81920,)\n",
            "551 TRNHNQS128E0784EDF (40256,)\n",
            "552 TRLAACO128F148FDC0 (14560,)\n",
            "553 TRYCYQE128E0792EC9 (85312,)\n",
            "554 TRBDKMV128F42A703F (83552,)\n",
            "555 TRTXAEL128E07930C6 (173376,)\n",
            "556 TRFLCMC128E0792ECE (199008,)\n",
            "557 TRRCMCP128E0792EC3 (63904,)\n",
            "558 TRGTVAI128F426AA6A (80352,)\n",
            "559 TRPMMGT128F4240960 (174336,)\n",
            "560 TRZKSEK128F426C8B0 (40256,)\n",
            "561 TRNHUEN128E07930B8 (133824,)\n",
            "562 TRXUVUR128F935A07A (280544,)\n",
            "563 TRFHGWV128E0792EC8 (27040,)\n",
            "564 TRSGPTZ128F931C5C1 (25344,)\n",
            "565 TRNCSKU128F4265639 (31168,)\n",
            "566 TRDIFHF128F422B9CD (187328,)\n",
            "567 TRBOYTH128EF34A24B (37568,)\n",
            "568 TRWCUAT128F92FD8B9 (224320,)\n",
            "569 TRIPQQA128E0791249 (178720,)\n",
            "570 TRJETZT128F933F3C9 (236640,)\n",
            "571 TRFQTXH128F934B72D (184480,)\n",
            "572 TRMCMTV128E078C118 (45120,)\n",
            "573 TRVDAHO128F92F231E (11744,)\n",
            "574 TRRUBZB128E078B30B (14688,)\n",
            "575 TRTFCFG128F42152C1 (54496,)\n",
            "576 TRYDUAU128F429825B (98624,)\n",
            "577 TRKZYKJ128EF35EDAC (11392,)\n",
            "578 TRQNDUZ128F42152CD (172224,)\n",
            "579 TRMLRJW128F4263C3C (80032,)\n",
            "580 TRFTOSV128E078DF78 (69696,)\n",
            "581 TRAAEEH128E0795DFE (199168,)\n",
            "582 TRNWGRU128F1460571 (120096,)\n",
            "583 TRZPETU128F427704B (51968,)\n",
            "584 TRJKMHK128F1457FAC (47584,)\n",
            "585 TROXTXS128F9331D60 (140832,)\n",
            "586 TRWAVSM128F428711F (50560,)\n",
            "587 TRCVCUS128F42838DD (52896,)\n",
            "588 TRLRTEM128F9331D68 (74464,)\n",
            "589 TRRQZKF128F147CEA0 (64448,)\n",
            "590 TRDUSVC128F4263C8D (8288,)\n",
            "591 TRJQXHD128F428E701 (164608,)\n",
            "592 TRFZJOZ128F4263BE3 (55072,)\n",
            "593 TRLSPZC128F4263C3A (190304,)\n",
            "594 TRIIALE128F92CD1BE (71872,)\n",
            "595 TRGOSCC128F42AC959 (37184,)\n",
            "596 TRATANL128F92E3FC5 (18304,)\n",
            "597 TRMFAWD128F932B11D (204448,)\n",
            "598 TRTLDJA128F14570E8 (139552,)\n",
            "599 TRHWGCZ128F42B328C (342048,)\n",
            "600 TRCESGY128E0780E09 (108640,)\n",
            "601 TRCUEPI128F4278E10 (122592,)\n",
            "602 TRBXZAK128F4263E56 (35008,)\n",
            "603 TRCXWLU128F92F2A0D (68896,)\n",
            "604 TRWWMWX128F428686C (21696,)\n",
            "605 TRHRCXK128F4263C4C (146304,)\n",
            "606 TRPFCIB128F14B026B (107104,)\n",
            "607 TRTBVXE128F932828E (134080,)\n",
            "608 TRUAJSP128F4273741 (125280,)\n",
            "609 TRMWEBJ128F931D025 (43776,)\n",
            "610 TRBRLWO128F4265309 (21952,)\n",
            "611 TRDDZMP128EF35F98E (186816,)\n",
            "612 TRHVBCC128F429E974 (21952,)\n",
            "613 TRWNTBN128F42727D7 (48480,)\n",
            "614 TRRVFSI128EF34A1AE (34752,)\n",
            "615 TRXOSUG128F93450BA (21824,)\n",
            "616 TROSQNP128F92C5073 (164352,)\n",
            "617 TREXJJE128E0781B1D (110368,)\n",
            "618 TRWMZOQ12903CC16CB (89792,)\n",
            "619 TRMFQYG128F4240D9E (27488,)\n",
            "620 TRUNQFP12903D08475 (198400,)\n",
            "621 TRHWCVS128F14895A3 (22400,)\n",
            "623 TRYFTOL128F92FDA1A (158432,)\n",
            "624 TRXTPLJ128F9314B45 (34048,)\n",
            "625 TRYZMOC128F423E58D (69792,)\n",
            "626 TRCSZFD12903CE7FDB (74720,)\n",
            "627 TRTNITP128F149D9B1 (236160,)\n",
            "628 TRDTVGI128E078EDBB (31648,)\n",
            "629 TRBTVVD128F146D742 (118016,)\n",
            "630 TRLPCKY128F42ACFB8 (62080,)\n",
            "631 TRXESGJ12903CC04D2 (171488,)\n",
            "632 TRACHRV128F4214FB6 (138848,)\n",
            "633 TRUKRLU128E07962F6 (57408,)\n",
            "634 TRUKNID128F933E4BC (97312,)\n",
            "635 TRADTWS128F4241AC1 (50976,)\n",
            "636 TRIZYOV128F4269D01 (381216,)\n",
            "637 TRAXLZU12903D05F94 (63232,)\n",
            "638 TRCOOYB128E078ED95 (13856,)\n",
            "639 TRVJVEG12903CD028D (122848,)\n",
            "640 TRAJFBN128F930527C (71264,)\n",
            "641 TRFGYZI128F4243BFF (85824,)\n",
            "642 TRIKEND12903CCC5BE (107552,)\n",
            "643 TRHNENS128E0799A33 (108448,)\n",
            "644 TRJNQAS128F42A6E02 (63488,)\n",
            "645 TRZGLBD128F427E2DC (113088,)\n",
            "646 TRKBKQE128C71969A2 (125248,)\n",
            "647 TRJXPVO12903CFB3FD (271968,)\n",
            "648 TRXNXPH128F93061FF (59840,)\n",
            "649 TRMMPDJ128F429472E (137248,)\n",
            "650 TRVGBKW128F931F66A (62944,)\n",
            "651 TRLICII128F427CF5B (196032,)\n",
            "652 TRMHTLI128F92F1339 (77600,)\n",
            "653 TRFHGSV128F4273973 (80768,)\n",
            "654 TRRKJYF128F9308958 (240448,)\n",
            "655 TRCJBGR128EF341FF3 (136128,)\n",
            "656 TRFWIGT128F147CC7B (52224,)\n",
            "657 TRCMQTP128F147CBD0 (85184,)\n",
            "658 TRSIOVA128F42850CF (31904,)\n",
            "659 TRJUFNY12903CD0B61 (103456,)\n",
            "660 TRZPVLJ128F148D2F7 (32800,)\n",
            "661 TROVGLQ128F4262EBD (112896,)\n",
            "662 TRVFFVR128E07856B5 (66016,)\n",
            "663 TRSHTER12903CE5606 (124960,)\n",
            "664 TRCRFLR128F422D98D (188320,)\n",
            "665 TRGHAJW12903CBF769 (100320,)\n",
            "666 TRLCNWM128F423BB63 (369408,)\n",
            "667 TRZPUFG128F427CC3B (246080,)\n",
            "668 TRPWMGN128E078F2B8 (58944,)\n",
            "669 TRXJRSJ128F9330D11 (30336,)\n",
            "670 TRFTLFA128F42606B5 (80736,)\n",
            "671 TRIROLS128F92F8E10 (83744,)\n",
            "672 TRGTIIJ128F424DA28 (24064,)\n",
            "673 TRMJZAU128F428268C (143616,)\n",
            "674 TRVFUNQ128F92CF00D (108224,)\n",
            "675 TRASRXK128F9311176 (22944,)\n",
            "676 TRTNJHH128F932A4DC (124352,)\n",
            "677 TREBYDR128F9342EF4 (218976,)\n",
            "678 TRVYSIX128F145A3CC (79168,)\n",
            "679 TRZHSMF128F146EC2F (231424,)\n",
            "680 TRUQTED12903CB7445 (148992,)\n",
            "681 TRBSQIW128F9306A32 (316544,)\n",
            "682 TRRIRMX128F92E6272 (45184,)\n",
            "683 TRROECL128F425C1C5 (79904,)\n",
            "684 TRGVDFE128F42A384F (151840,)\n",
            "685 TRPFEPI128F147C7DA (142272,)\n",
            "686 TREKLEO128F1459433 (724256,)\n",
            "687 TRQOOUE128E078253D (39104,)\n",
            "688 TRJHXEO128F425C34E (135872,)\n",
            "689 TRJRMAE128F423EEE1 (12480,)\n",
            "690 TRWQAFX128F428DBB0 (141952,)\n",
            "691 TRCMTEO128F931DBA8 (4480,)\n",
            "692 TRKRBIA128F145C03A (34752,)\n",
            "693 TRPSLCW128F9301F83 (70080,)\n",
            "694 TRZSOWU128F9310EA2 (54912,)\n",
            "695 TRGKLSE12903CCC5CD (38976,)\n",
            "696 TRLWTAG128E0789037 (31616,)\n",
            "697 TRLDPKS128E078DC7F (83904,)\n",
            "698 TRMBYTX12903CBB8A7 (51936,)\n",
            "699 TRNUMRY12903CB6530 (27456,)\n",
            "700 TRHNGJL128F9310B69 (150240,)\n",
            "701 TRHCGUO12903CB28D7 (173184,)\n",
            "702 TRGOAJI128F146E0AD (160896,)\n",
            "703 TRIOOIT128F14910F7 (106912,)\n",
            "704 TRDKEWM128F145267A (108416,)\n",
            "705 TRASUWA128F933E6F7 (212512,)\n",
            "706 TRJPJFB128F426779F (428416,)\n",
            "707 TRDENBR128E0786BFA (63200,)\n",
            "708 TRALSBI12903CCEEA0 (217344,)\n",
            "709 TRIVSGU128F14592BB (46656,)\n",
            "710 TRYADUY128F93176B8 (100064,)\n",
            "711 TRDSHZG128E0784128 (71904,)\n",
            "712 TRCKBMN128F93583E1 (193632,)\n",
            "713 TRFMWJL128F4257AF1 (209984,)\n",
            "714 TRAOEDK128E07814F1 (277760,)\n",
            "715 TRZRPDM128F4298331 (181376,)\n",
            "716 TRFZMNE128F426D9AE (21184,)\n",
            "717 TRIODZU128E078F3E2 (157824,)\n",
            "718 TRKECHT128F93069D2 (95488,)\n",
            "719 TRIUJRC128F14A5547 (25216,)\n",
            "720 TRDBZRW128F932B709 (67616,)\n",
            "721 TRARREW128F92CD3D9 (101760,)\n",
            "722 TRVFHPK12903CA720C (168672,)\n",
            "723 TRTWEJL128E078DCCA (24608,)\n",
            "724 TRGSBNZ12903CD668D (13504,)\n",
            "725 TRONCAS128F931E2A3 (116256,)\n",
            "726 TRHJOFO12903CBFDE9 (77952,)\n",
            "727 TRIUFXI128E0788E4A (31744,)\n",
            "728 TRZSLUR128E0782FA5 (196672,)\n",
            "729 TRFUOZN128F4297F8C (161888,)\n",
            "730 TRYFBEP128F427E7E2 (180064,)\n",
            "731 TRJEPES128F92C577B (65312,)\n",
            "732 TRPEAEG128F1458AAB (51264,)\n",
            "733 TRGDLYG12903CB0D32 (38880,)\n",
            "734 TREINNG128F4284976 (69664,)\n",
            "735 TRQLPVS128F4296070 (105824,)\n",
            "736 TRYYOWT128F42B6558 (42592,)\n",
            "737 TRKMXSU12903CCC6F0 (116896,)\n",
            "738 TRZUOBA128F425D72D (135840,)\n",
            "739 TRRCXDT128F426050E (39648,)\n",
            "740 TRTIQHM12903CC98F7 (160640,)\n",
            "741 TRGPTPB128F92F64D7 (92544,)\n",
            "742 TRBSMHW128F92E385A (15648,)\n",
            "743 TRWKSNO128F9326746 (243968,)\n",
            "744 TRCFQHK12903CB0D57 (44256,)\n",
            "745 TRQGQQD128EF35EF3E (31424,)\n",
            "746 TRJOWRR128F423A09F (54624,)\n",
            "747 TRVHWUW128F42155B0 (51904,)\n",
            "748 TRWWMBK128F4261DFE (74464,)\n",
            "749 TREVKGL128EF35B86A (94848,)\n",
            "750 TRMSAZU128F428CAF3 (62848,)\n",
            "751 TRCGPCB128F42BA5D2 (74944,)\n",
            "752 TRZTYVR128F42486EA (124192,)\n",
            "753 TRYRIVM128E078A7A5 (144992,)\n",
            "754 TRANSKF128F147FCB4 (118176,)\n",
            "755 TRODNCQ128EF342E37 (116320,)\n",
            "756 TRMCUTS128F1495E17 (101120,)\n",
            "757 TRJQBXY128E078FD6D (176128,)\n",
            "758 TRAWDRR128F4285F57 (50592,)\n",
            "759 TRSIVFN12903CCB281 (111584,)\n",
            "760 TREOPTO128F1485D8B (134944,)\n",
            "761 TRAYQPE128F42ACB09 (31520,)\n",
            "762 TRPQWHY12903CD94FB (75520,)\n",
            "763 TRNSXCK12903C9AE4B (51360,)\n",
            "764 TRVBNPA128F92F37CD (144640,)\n",
            "765 TRHDHNG128F9305C1E (85472,)\n",
            "766 TRNWEYN128E0793C6E (137184,)\n",
            "767 TRQEMQL128F14632E1 (167328,)\n",
            "768 TRWOLMX128EF33FB7D (211264,)\n",
            "769 TRSBFIQ128F145B050 (18336,)\n",
            "770 TRZNMWM12903CCE385 (327008,)\n",
            "771 TRAXEOC128F422B0B3 (183808,)\n",
            "772 TRFJWFT128F1484D63 (8192,)\n",
            "773 TRPZUFD128F4273C81 (329856,)\n",
            "774 TRPUYLI128F425D72A (258912,)\n",
            "775 TRPTFGM128F428A15A (187200,)\n",
            "776 TRFZQIW128F93363FC (427904,)\n",
            "777 TRARIBY128F4273287 (20832,)\n",
            "778 TRYRHRM128F148B577 (61568,)\n",
            "779 TRWQQJK128F1495E02 (248992,)\n",
            "780 TRMPISD12903D05F90 (101120,)\n",
            "781 TRZPFTH128F428E7CF (307840,)\n",
            "782 TRBVMHI128F428A5B1 (37536,)\n",
            "783 TRPBRGV128F930981A (36416,)\n",
            "784 TRRHSQS128F42900FF (28704,)\n",
            "785 TRKLSHC128F92DD46E (119680,)\n",
            "786 TRSOBXT128F1459EB4 (51296,)\n",
            "787 TRRJQGX128E078A473 (32672,)\n",
            "788 TRMYQZR128F4254E39 (72032,)\n",
            "789 TRAUVUF128F148D1AA (46624,)\n",
            "790 TRBWQED128F145C202 (408864,)\n",
            "791 TRMJVYS128F932084D (42176,)\n",
            "792 TRAFMVN128F147CBCE (86880,)\n",
            "793 TRKHDXT128EF342E3D (37632,)\n",
            "794 TRQWBKW128EF343994 (39584,)\n",
            "795 TRXHMVJ128F149CB95 (32288,)\n",
            "796 TRTUSHE128F1464844 (243584,)\n",
            "797 TRSSSJW128F146D5DC (33728,)\n",
            "798 TRDZMQG128F149CB9B (36928,)\n",
            "799 TRKKNPC128F9332C43 (22848,)\n",
            "800 TRJMBZA128F4263BB8 (218624,)\n",
            "801 TRFITXC128F1469028 (152288,)\n",
            "802 TRHTQFK128F427BBAA (46016,)\n",
            "803 TRDFLOV128F93509C5 (179072,)\n",
            "804 TRVARTJ128F425AF90 (178080,)\n",
            "805 TRCZHQJ128F42AE2BF (158336,)\n",
            "806 TRALDUE128F9312AF2 (11680,)\n",
            "807 TRILFEK12903C9AAB7 (47552,)\n",
            "808 TRNJQAM128F14557AF (123232,)\n",
            "809 TRWQINA128F9339E57 (22496,)\n",
            "810 TRGETHY128F92D31F6 (40448,)\n",
            "811 TRAZDPO128E078ECE6 (37184,)\n",
            "812 TRLXPMB128EF342C12 (346720,)\n",
            "813 TRYATJD128F92FDB02 (140576,)\n",
            "814 TRXFBNU12903CCBE8A (33824,)\n",
            "815 TRBRPQK128E07936CD (47008,)\n",
            "816 TRQCWIN128F1456FB8 (91232,)\n",
            "817 TRWXDZF128F92F59A3 (378944,)\n",
            "818 TRJYWOA12903CCF332 (166048,)\n",
            "819 TRDCOOG128F42596E1 (124896,)\n",
            "820 TRXTHQI128F14623D7 (36128,)\n",
            "821 TRTOAMC128E0793521 (9792,)\n",
            "822 TRSRPMS128F92C478E (7264,)\n",
            "823 TRGJWNK128F9351E94 (153952,)\n",
            "824 TRJYKWF128E0784DA2 (221248,)\n",
            "825 TRGUQSQ128F425F6A3 (425344,)\n",
            "826 TRKSKAH128F42957F4 (44224,)\n",
            "827 TRFGQNT128F146B854 (217120,)\n",
            "828 TRENHWK128E078ED26 (25216,)\n",
            "829 TRAUWUX128F92FE41C (20896,)\n",
            "830 TRFFUKH128F92E0601 (48288,)\n",
            "831 TRKJDSS128F4262A09 (69536,)\n",
            "832 TRSNTXV128F4233580 (70528,)\n",
            "833 TRSSESO128F42A1E57 (157600,)\n",
            "834 TRVEBWL128F933B4DA (101920,)\n",
            "835 TRSEKGD128F42B654D (38336,)\n",
            "836 TRHZERX128E0783F5D (188000,)\n",
            "837 TRQZCUX128E079136D (44608,)\n",
            "838 TRSLNMO128F145F945 (23712,)\n",
            "839 TRVQFLX128F932A67D (78912,)\n",
            "840 TRWTFEA128F426E1DC (40096,)\n",
            "841 TRXTOSU128F42791A4 (38400,)\n",
            "842 TRXSOYO128F9343CC5 (281504,)\n",
            "843 TRKQGMV128F426DE6E (20992,)\n",
            "844 TRGTDXG128F92FE060 (88832,)\n",
            "845 TRZNEAW128F42779C4 (20352,)\n",
            "846 TRJHPJS128F424A9CA (163392,)\n",
            "847 TRFLAPP12903C9AACB (100704,)\n",
            "848 TRTNKXC128F4270EDC (180800,)\n",
            "849 TRPWCBK128F42BA1B1 (57056,)\n",
            "850 TRNJELH128F429525D (49088,)\n",
            "851 TRVIQZG128F14A8854 (94592,)\n",
            "852 TRCLZFC128E07928AC (43392,)\n",
            "853 TRKXICW12903C9AAB1 (173920,)\n",
            "854 TRQAFDS128F424A9BA (281120,)\n",
            "855 TRWSNUJ128F92D3C71 (51712,)\n",
            "856 TRTQHDD128F42886D1 (196320,)\n",
            "857 TRBXGHM128E0791287 (12128,)\n",
            "858 TRWRWED12903CB23E1 (41536,)\n",
            "859 TRMFPOH128F4273C6B (152736,)\n",
            "860 TRLUEZG12903C9AB9F (184000,)\n",
            "861 TRFXTLP128F424A9B6 (62496,)\n",
            "862 TRCAFIC128F424A9C0 (182240,)\n",
            "863 TRDQFMQ128F14745DD (43360,)\n",
            "864 TRRXMOY12903CB89CE (49504,)\n",
            "865 TRSJXRC128F42A4CDF (20544,)\n",
            "866 TRMSHWV128F92D2D67 (309568,)\n",
            "867 TRGELDM128F92EF5AA (114656,)\n",
            "868 TRHGXYQ128F148D328 (98816,)\n",
            "869 TRWBSOX128F92D2DCD (118048,)\n",
            "870 TRKEOKC12903CE7554 (154944,)\n",
            "871 TRQRKGB128E078122A (55968,)\n",
            "872 TRDGUKM128F423458E (132960,)\n",
            "873 TRIDJFI12903CC2F56 (38016,)\n",
            "874 TROYQUG128F146D754 (134336,)\n",
            "875 TRUYHHZ128EF341D29 (46880,)\n",
            "876 TRTXSVR128F426A371 (35808,)\n",
            "877 TRMOZVH128F148D323 (86592,)\n",
            "878 TRUJMVI128F42727D8 (57728,)\n",
            "879 TROXWBT128F9305E67 (188992,)\n",
            "880 TRNDFGY128F148D04D (153952,)\n",
            "881 TRBHUSP128F4242DD0 (233408,)\n",
            "882 TRZUIME128F427D742 (200160,)\n",
            "883 TRJJWLJ128F428AD88 (336768,)\n",
            "884 TRPRPXS128E078D012 (165824,)\n",
            "885 TRZECES128F9335E61 (325248,)\n",
            "886 TRQCMEL128F9358E17 (23808,)\n",
            "887 TRBZFDK128EF35A246 (28832,)\n",
            "888 TRHGEMF128F932E13C (134656,)\n",
            "889 TRQRCBM128F14640C5 (84800,)\n",
            "890 TRUVCLI12903C9AF36 (19776,)\n",
            "891 TRCFNZR12903CD78B9 (169184,)\n",
            "892 TREQAYI128F9305E5F (159968,)\n",
            "893 TRXVRTD128F427D420 (258528,)\n",
            "894 TRBMXRJ12903D07304 (189696,)\n",
            "895 TRIHNSZ12903C9FF9D (23808,)\n",
            "896 TRMWEOC128F9322C97 (17888,)\n",
            "897 TRLMGFC12903CF16BF (6624,)\n",
            "898 TRKLTEO12903CA15A9 (135392,)\n",
            "899 TRYFUFY128F92D7F01 (119936,)\n",
            "900 TRRSZCO128F4270273 (47232,)\n",
            "901 TRRCOIO128F92DCB64 (356928,)\n",
            "902 TREUEIN128F92E81DB (191744,)\n",
            "903 TRCHEPH128F934B901 (76672,)\n",
            "904 TROXXYE128F4267245 (98528,)\n",
            "905 TRVNYSZ128E0792A2B (46464,)\n",
            "906 TREQYBA12903CF16CE (640,)\n",
            "907 TRAFSAP128E0780ECF (222752,)\n",
            "908 TRRXGIS128E0781C66 (52256,)\n",
            "909 TRBCQSK128F92DF8BB (265632,)\n",
            "910 TRSSZGF128F42910AF (98240,)\n",
            "911 TRWVVMX128F1460888 (83968,)\n",
            "912 TRBXFBB12903CDF91C (197408,)\n",
            "913 TRVMCTC128F92CA4DD (305312,)\n",
            "914 TRSRYBA12903CA4A3D (61792,)\n",
            "915 TRYKRYI128F92FDC13 (154720,)\n",
            "916 TRSRRID128F92EF86C (64480,)\n",
            "917 TRXHJEJ128F422A3E0 (188768,)\n",
            "918 TROACYW128F423D2B4 (17728,)\n",
            "919 TROVEVL128F932A447 (68064,)\n",
            "920 TRMJUPD128F92E7217 (242720,)\n",
            "921 TRCCVSJ128F934B8DD (70976,)\n",
            "922 TRTWBNZ128F92F3426 (174432,)\n",
            "923 TROOWUH128F4238B6B (86240,)\n",
            "924 TRCCPTC128F14971B1 (22144,)\n",
            "925 TRLHLNL128F426BD1E (112256,)\n",
            "926 TRMLSSE128F4275DE9 (397984,)\n",
            "927 TRKDHRK128F426B40F (205216,)\n",
            "928 TRNPZWV128E0788BDA (75808,)\n",
            "929 TRRLZDT128F149DB42 (89472,)\n",
            "930 TRNSVGW128F424E127 (26528,)\n",
            "931 TRMJSWT12903CA8AE1 (69216,)\n",
            "932 TRHPKRN128F424EF48 (199168,)\n",
            "933 TRVYVRC128F9324F4E (58944,)\n",
            "934 TRXWPCY128F92CBEAA (16576,)\n",
            "935 TRQZHAY128F424E1C4 (201760,)\n",
            "936 TRGWMEB128F92EB234 (25152,)\n",
            "937 TRHPFJR12903CA41DA (107040,)\n",
            "938 TRRKIKF128F92FE98F (50176,)\n",
            "939 TRLZGLA128F423E3F6 (121728,)\n",
            "940 TRJDTBE128F4289421 (49312,)\n",
            "941 TRBQNCM12903CD8BA0 (42656,)\n",
            "942 TRSKWQX128F42A88D2 (154208,)\n",
            "943 TRXJYEO128F426AD39 (17568,)\n",
            "944 TRZUWSX128F92F491A (89632,)\n",
            "945 TRRKGRC128F932D8F0 (46016,)\n",
            "946 TRKIZWL128EF342C5A (192832,)\n",
            "947 TRBBEVP128F146ECF8 (81536,)\n",
            "948 TRFIWDT128EF35EA16 (20384,)\n",
            "949 TRAVZEK128E078F2DB (80352,)\n",
            "950 TRKEYEF128F92EB05B (145504,)\n",
            "951 TROVCDV128F931D976 (73280,)\n",
            "952 TRQNONI128F428C73B (103168,)\n",
            "953 TRMJXVS12903CACEDC (215616,)\n",
            "954 TRJBNZY128EF35A247 (145696,)\n",
            "955 TRMNFVE128F932907E (33184,)\n",
            "956 TROPAHQ128F93205A6 (60960,)\n",
            "957 TRGZHAP128F425AEF8 (181312,)\n",
            "958 TRJDQUN128E078F415 (32032,)\n",
            "959 TRPWEWO128F424F0D2 (85920,)\n",
            "960 TRXUBFV128F42410C4 (52128,)\n",
            "961 TRXHSAE128E078DEF2 (96960,)\n",
            "962 TRYLNMC12903CED275 (13280,)\n",
            "963 TRSKYGY12903CCC681 (34144,)\n",
            "964 TRLOBFL128F4278F0D (84064,)\n",
            "965 TRTYXZK128F4290090 (11648,)\n",
            "966 TRTZWXH128F92DC8BD (33952,)\n",
            "967 TRWPXOX128F93465C6 (29728,)\n",
            "968 TRFBARJ128F42480A0 (127648,)\n",
            "969 TRJVZFI128F92E6CEC (89184,)\n",
            "970 TRXPADO128F42A964A (59840,)\n",
            "971 TRULNMQ128F92E1FDA (45792,)\n",
            "972 TRADYQK128F42A762A (73920,)\n",
            "973 TRUUQTZ128E07869CC (115296,)\n",
            "974 TRNOHXY128F930FB6B (114048,)\n",
            "975 TRSRCZC128F9314C2F (114944,)\n",
            "976 TRIAQLQ128E078A5A3 (39264,)\n",
            "977 TRXPNTR128E078662C (197440,)\n",
            "978 TRQHEGS128E0786C55 (55936,)\n",
            "979 TRENNLG128F934324D (125120,)\n",
            "980 TRLBDBO128F14531F2 (171776,)\n",
            "981 TRHXKMH128E0786BA3 (296544,)\n",
            "982 TRCPTQP128F423A80E (103648,)\n",
            "983 TRQXLCM128F429B770 (239008,)\n",
            "984 TRYPSFO12903CA84F0 (50464,)\n",
            "985 TRCISNM128E0785E13 (214784,)\n",
            "986 TRFZZXF128F92EF279 (35360,)\n",
            "987 TRHVOHR128E07840A4 (86944,)\n",
            "988 TROQOXI128F42734EB (331328,)\n",
            "989 TRZRDDD128F4236EB1 (65440,)\n",
            "990 TRVZDQB128F92F2C61 (29504,)\n",
            "991 TRXTXMP128EF35F7B9 (125824,)\n",
            "992 TRQIESP128F428CA9D (58176,)\n",
            "993 TRKWZNC128F4281DBA (64960,)\n",
            "994 TRQFEEL128F147C208 (53248,)\n",
            "995 TRCEMNR128F92FE2EE (81280,)\n",
            "996 TRWJYCV12903CF07ED (31296,)\n",
            "997 TRWAMGI128F422B7ED (31648,)\n",
            "998 TRJFBHM128F426712A (254880,)\n",
            "999 TRZKDAQ128F149E20B (74688,)\n",
            "1000 TRIISEF12903CDFAAF (155232,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T9dSYpIArBcm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_sample.to_csv(PATH + 'df_sample.csv', index=False)\n",
        "# with open(PATH + 'tracks_sample.json', 'w') as fp:\n",
        "#     json.dump(tracks, fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z5QrcWplvYVe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(PATH + 'tracks_sample.pkl', 'rb') as fp:\n",
        "    testing = pickle.load(fp)  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpyHoKvjv0Fw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a33e4bad-651c-4433-867d-10f09ca130f6"
      },
      "source": [
        "len(testing.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yt_tLREM3H1M",
        "colab_type": "text"
      },
      "source": [
        "# STAGE 1.3 N songs sample data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ILUYYg3csM_0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 344
        },
        "outputId": "6dac83fb-d470-46c7-e3dd-2ec2abd6ec54"
      },
      "source": [
        "df = pd.read_csv(PATH + 'df2_.csv')\n",
        "print(df.shape, df['user'].unique().shape, df['track'].unique().shape)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1318132, 7) (32126,) (9529,)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "      <th>midi_exists</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOANQFY12AB0183239</td>\n",
              "      <td>1</td>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOAYATB12A6701FD50</td>\n",
              "      <td>1</td>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBOAFP12A8C131F36</td>\n",
              "      <td>7</td>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBZZDU12A6310D8A3</td>\n",
              "      <td>7</td>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SODEAWL12AB0187032</td>\n",
              "      <td>8</td>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                                              paths\n",
              "0           0  ...  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1           1  ...  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2           2  ...  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3           3  ...  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4           4  ...  /content/drive/My\\ Drive/dataset/lmd_matched/Q...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vilvSzI5sR4Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(PATH + 'tracks_sample.pkl', 'rb') as fp:\n",
        "    songs_sample = pickle.load(fp)  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZcnAPdWshho",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 344
        },
        "outputId": "b384a5e4-3412-4fb4-b35f-847e97910c31"
      },
      "source": [
        "sample_tracks = list(songs_sample.keys())\n",
        "df_sample = df[df['track'].isin(sample_tracks)]\n",
        "print(df_sample.shape, df_sample['user'].unique().shape, df_sample['track'].unique().shape)\n",
        "df_sample.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(888373, 7) (32126,) (1000,)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user</th>\n",
              "      <th>song</th>\n",
              "      <th>play_count</th>\n",
              "      <th>track</th>\n",
              "      <th>midi_exists</th>\n",
              "      <th>paths</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOANQFY12AB0183239</td>\n",
              "      <td>1</td>\n",
              "      <td>TROMKCG128F9320C09</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/O...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOAYATB12A6701FD50</td>\n",
              "      <td>1</td>\n",
              "      <td>TREVCGZ128E079232A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/E...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBOAFP12A8C131F36</td>\n",
              "      <td>7</td>\n",
              "      <td>TRCPAGR128F423A01A</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SOBZZDU12A6310D8A3</td>\n",
              "      <td>7</td>\n",
              "      <td>TRGWWRE128E0781CDE</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/G...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
              "      <td>SODEAWL12AB0187032</td>\n",
              "      <td>8</td>\n",
              "      <td>TRQRVQG12903CAA7BD</td>\n",
              "      <td>True</td>\n",
              "      <td>/content/drive/My\\ Drive/dataset/lmd_matched/Q...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  ...                                              paths\n",
              "0           0  ...  /content/drive/My\\ Drive/dataset/lmd_matched/O...\n",
              "1           1  ...  /content/drive/My\\ Drive/dataset/lmd_matched/E...\n",
              "2           2  ...  /content/drive/My\\ Drive/dataset/lmd_matched/C...\n",
              "3           3  ...  /content/drive/My\\ Drive/dataset/lmd_matched/G...\n",
              "4           4  ...  /content/drive/My\\ Drive/dataset/lmd_matched/Q...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zEwAr_q_tf9T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145
        },
        "outputId": "f046abdb-c910-4636-a3f9-69030d75363b"
      },
      "source": [
        "def rating(row):\n",
        "  if row['play_count'] <= 1:\n",
        "    return 1\n",
        "  elif row['play_count'] == 2:\n",
        "    return 2\n",
        "  elif row['play_count'] == 3:    \n",
        "    return 3\n",
        "  elif (row['play_count'] >= 4) and (row['play_count'] <= 5):\n",
        "    return 4\n",
        "  return 5    \n",
        "\n",
        "df_sample['rating'] = df_sample.apply(rating, axis=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  if sys.path[0] == '':\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4OSXp4MsHxi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "user_to_id = {}\n",
        "track_to_id = {}\n",
        "\n",
        "for index, name in enumerate(df_sample['user'].unique()):\n",
        "  user_to_id[name] = index + 1\n",
        "\n",
        "for index, name in enumerate(df_sample['track'].unique()):  \n",
        "  track_to_id[name] = index + 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFu_kMqAZe0h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "id_to_user = {}\n",
        "id_to_track = {}\n",
        "\n",
        "for k, v in user_to_id.items():\n",
        "    id_to_user[v] = k\n",
        "\n",
        "for k, v in track_to_id.items():\n",
        "    id_to_track[v] = k"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GyTqHzrtSm6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 253
        },
        "outputId": "efb6d36c-af74-4e13-c5f1-a5064bba74a3"
      },
      "source": [
        "df_sample['user_id'] = df_sample['user'].map(user_to_id)\n",
        "df_sample['track_id'] = df_sample['track'].map(track_to_id)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJ-MQa9DsJWH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(PATH + 'user_to_id.json', 'w') as f:\n",
        "    json.dump(user_to_id, f, indent=4)\n",
        "\n",
        "with open(PATH + 'track_to_id.json', 'w') as f:\n",
        "    json.dump(track_to_id, f, indent=4)    \n",
        "\n",
        "with open(PATH + 'id_to_user.json', 'w') as f:\n",
        "    json.dump(id_to_user, f, indent=4)    \n",
        "\n",
        "with open(PATH + 'id_to_track.json', 'w') as f:\n",
        "    json.dump(id_to_track, f, indent=4)        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SpCN521TtP6r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "f8388ab4-c77c-4367-ad06-40c50865753b"
      },
      "source": [
        "# user to int and song to int mapping.\n",
        "df_sample_feat = df_sample[['user_id', 'track_id', 'rating']]\n",
        "df_sample_feat.to_csv(PATH + 'df_1k.csv', index=False)\n",
        "df_sample_feat.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>track_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id  track_id  rating\n",
              "0        1         1       1\n",
              "1        1         2       1\n",
              "2        1         3       5\n",
              "3        1         4       5\n",
              "4        1         5       5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    }
  ]
}