preprocess-tts-texts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d92f8ff7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://gist.githubusercontent.com/huseinzol05/98974ae8c6c7a65d4bc0af9f5003786a/raw/2e06e71ef7349a57bc58cc9913ae6bae1f9f8447/mp.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "faafabe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "import mp\n",
    "from tqdm import tqdm\n",
    "from datasets import Dataset\n",
    "from phonemize import phonemize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be9d2ee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "\n",
    "snapshot_download(\n",
    "    repo_id='mesolitica/TTS', repo_type='dataset', \n",
    "    allow_patterns = 'texts/*.json', local_dir = './')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "acd4b111",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "config_path = \"Configs/config.yml\"\n",
    "config = yaml.safe_load(open(config_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fcee0ffd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import phonemizer\n",
    "global_phonemizer = phonemizer.backend.EspeakBackend(language='ms', preserve_punctuation=True,  with_stress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4961cd3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(config['dataset_params']['tokenizer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b4e05bf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for f in glob('texts/*.json'):\n",
    "    with open(f) as fopen:\n",
    "        d = json.load(fopen)\n",
    "    data.extend(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "23b1a49c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "272785"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "07de9ca9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'original': 'Untuk menangani isu rasuah dan akauntabiliti dalam politik Malaysia, pelbagai pembaharuan dan langkah boleh diambil.',\n",
       " 'normalized': 'Untuk menangani isu rasuah dan akauntabiliti dalam politik Malaysia , pelbagai pembaharuan dan langkah boleh diambil .'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fad7b25f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1205177"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "731799f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [26,\n",
       "  3167,\n",
       "  870,\n",
       "  2624,\n",
       "  9,\n",
       "  21057,\n",
       "  18,\n",
       "  228,\n",
       "  43,\n",
       "  5,\n",
       "  258,\n",
       "  2383,\n",
       "  9,\n",
       "  1127,\n",
       "  97,\n",
       "  967,\n",
       "  6],\n",
       " 'phonemes': ['ˈuntoʔ',\n",
       "  'mənaŋˈani',\n",
       "  'ˈisu',\n",
       "  'rasˈuah',\n",
       "  'dan',\n",
       "  'ˌakaʊntˌabilˈiti',\n",
       "  'dˈalam',\n",
       "  'polˈiteʔ',\n",
       "  'məlˈesiə',\n",
       "  ',',\n",
       "  'pəlbˈaɡaɪ',\n",
       "  'pəmbˌaharˈuan',\n",
       "  'dan',\n",
       "  'lˈaŋkah',\n",
       "  'bˈoleh',\n",
       "  'diˈambel',\n",
       "  '.']}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phonemize(data[0]['normalized'], global_phonemizer, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cda7d83d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loop(data):\n",
    "    data, _ = data\n",
    "    results = []\n",
    "    for d in tqdm(data):\n",
    "        r = phonemize(d['normalized'], global_phonemizer, tokenizer)\n",
    "        r['id'] = '1'\n",
    "        r['url'] = ''\n",
    "        r['title'] = ''\n",
    "        results.append(r)\n",
    "    return results\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d1257be6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:00<00:00, 688.88it/s]\n"
     ]
    }
   ],
   "source": [
    "results = loop((data[:10], 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7d46ed5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "  4%|███▏                                                                               | 2343/60258 [00:05<04:06, 235.01it/s]\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      " 33%|███████████████████████████                                                       | 19872/60258 [02:51<06:03, 110.99it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      " 44%|███████████████████████████████████▋                                              | 26221/60258 [02:55<04:30, 125.85it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      " 47%|██████████████████████████████████████▉                                           | 28611/60258 [03:11<03:31, 149.97it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      " 70%|█████████████████████████████████████████████████████████▌                        | 42314/60258 [03:38<01:31, 196.76it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      " 56%|██████████████████████████████████████████████▏                                   | 33951/60258 [03:47<03:41, 118.72it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [05:09<00:00, 194.66it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████| 17/17 [00:00<00:00, 44.66it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [05:19<00:00, 188.79it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [05:19<00:00, 188.68it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [05:23<00:00, 186.06it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [05:35<00:00, 179.63it/s]\n",
      " 66%|██████████████████████████████████████████████████████▎                           | 39895/60258 [05:42<02:43, 124.76it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [05:33<00:00, 180.77it/s]\n",
      " 89%|█████████████████████████████████████████████████████████████████████████▎        | 53917/60258 [05:48<00:27, 233.36it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      " 93%|████████████████████████████████████████████████████████████████████████████▎     | 56081/60258 [06:04<00:13, 313.35it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:19<00:00, 158.84it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:17<00:00, 159.57it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:25<00:00, 156.14it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:24<00:00, 156.89it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:35<00:00, 152.44it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:34<00:00, 152.69it/s]\n",
      " 87%|██████████████████████████████████████████████████████████████████████▉           | 52135/60258 [06:37<00:29, 272.54it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:56<00:00, 144.55it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:56<00:00, 144.68it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:55<00:00, 145.12it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [06:57<00:00, 144.18it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [07:11<00:00, 139.52it/s]\n",
      " 98%|████████████████████████████████████████████████████████████████████████████████▍ | 59105/60258 [07:11<00:04, 271.53it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      " 95%|█████████████████████████████████████████████████████████████████████████████▊    | 57146/60258 [07:13<00:10, 287.95it/s]/home/husein/.local/lib/python3.10/site-packages/malaya/normalizer/rules.py:424: FutureWarning: Possible nested set at position 3\n",
      "  if re.findall(_expressions['ic'], word_lower):\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [07:16<00:00, 138.08it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [07:07<00:00, 140.82it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 60258/60258 [07:23<00:00, 135.94it/s]\n"
     ]
    }
   ],
   "source": [
    "results = mp.multiprocessing(data, loop, cores = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b6ff4346",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = Dataset.from_list(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2fd4a7ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "222cde301b3e41d0937ccfe40223ab63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Saving the dataset (0/1 shards):   0%|          | 0/1205177 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset.save_to_disk('news')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "2f57a630",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [26,\n",
       "  3167,\n",
       "  870,\n",
       "  2624,\n",
       "  9,\n",
       "  21057,\n",
       "  18,\n",
       "  228,\n",
       "  43,\n",
       "  5,\n",
       "  258,\n",
       "  2383,\n",
       "  9,\n",
       "  1127,\n",
       "  97,\n",
       "  967,\n",
       "  6],\n",
       " 'phonemes': ['ˈuntoʔ',\n",
       "  'mənaŋˈani',\n",
       "  'ˈisu',\n",
       "  'rasˈuah',\n",
       "  'dan',\n",
       "  'ˌakaʊntˌabilˈiti',\n",
       "  'dˈalam',\n",
       "  'polˈiteʔ',\n",
       "  'məlˈesiə',\n",
       "  ',',\n",
       "  'pəlbˈaɡaɪ',\n",
       "  'pəmbˌaharˈuan',\n",
       "  'dan',\n",
       "  'lˈaŋkah',\n",
       "  'bˈoleh',\n",
       "  'diˈambel',\n",
       "  '.'],\n",
       " 'id': '1',\n",
       " 'url': '',\n",
       " 'title': ''}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "047becbd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3.10",
   "language": "python",
   "name": "python3.10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}