Multiple_file_plotting_and_CSV.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import sounddevice as sd\n",
    "from matplotlib import pyplot as plt\n",
    "from datetime import datetime\n",
    "from scipy.io.wavfile import write"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving all files in a folder to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "give_input_folder_path_here=str(os.getcwd())+'\\\\DATA'\n",
    "give_output_CSV_folder_path_here=str(os.getcwd())+'\\\\DATA\\\\CSV'\n",
    "give_output_Wav_folder_path_here=str(os.getcwd())+'\\\\DATA\\\\Wav'\n",
    "date_format = '%Y-%m-%d %H:%M:%S.%f' \n",
    "# Change  date_format according to the value you get because the timestamp is generated by the server and may vary for every server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "if (os.path.exists(give_output_CSV_folder_path_here)==False):\n",
    "    os.mkdir(give_output_CSV_folder_path_here)\n",
    "if (os.path.exists(give_output_Wav_folder_path_here)==False):\n",
    "    os.mkdir(give_output_Wav_folder_path_here)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample Rate of microphone for Sample1 are: 8122.023832702967\n",
      "Sample Rate of microphone for Sample2 are: 8147.315362323815\n"
     ]
    }
   ],
   "source": [
    "for filename in glob.glob(os.path.join(give_input_folder_path_here, '*.txt')):\n",
    "    with open(os.path.join(os.getcwd(), filename), 'r') as f:\n",
    "        f_name=filename.split('\\\\')[-1]\n",
    "        f_name=f_name.split('.')[0]\n",
    "        aa=str(filename[:-4])\n",
    "        string=f.read()\n",
    "        string=string.replace(\"\\x00\",\"\")\n",
    "        pattern = \"\\[(.*?)\\]\"\n",
    "        timestamps = re.findall(pattern, string)\n",
    "        totaltime=datetime.strptime(timestamps[-1], date_format)-datetime.strptime(timestamps[0], date_format)\n",
    "        timeinsec=totaltime.total_seconds()\n",
    "        pattern = \"\\((.*?)\\)\"\n",
    "        Microphone_temp = re.findall(pattern, string)\n",
    "        Microphone_temp=[re.sub('\\[(.*?)\\]', '', x) for x in Microphone_temp]\n",
    "        Microphone=[x for x in Microphone_temp if len(x)<12]\n",
    "        Microphone_numpy=np.array(Microphone)\n",
    "        Microphone_numpy = Microphone_numpy.astype(int)\n",
    "        Sr=len(Microphone_temp)/timeinsec\n",
    "        print(\"Sample Rate of microphone for {} are: {}\".format(f_name,Sr))\n",
    "        sound_df=pd.DataFrame(Microphone_numpy,columns=['Microphone'])\n",
    "        sound_df.to_csv(give_output_CSV_folder_path_here+\"\\\\\"+f_name+'_Microphone.csv',encoding='utf-8')\n",
    "        scaled = np.int16(Microphone_numpy/np.max(np.abs(Microphone_numpy)) * 32767)\n",
    "        write(give_output_Wav_folder_path_here+\"\\\\\"+f_name+'_Microphone.wav', int(Sr), scaled)\n",
    "        pattern = \"\\|(.*?)\\|\"\n",
    "        temperature = re.findall(pattern, string)\n",
    "        temperature_numpy=np.array(temperature)\n",
    "        temperature_numpy = temperature_numpy.astype(float)\n",
    "        pattern = \"\\!(.*?)\\!\"\n",
    "        humidity = re.findall(pattern, string)\n",
    "        humidity_numpy=np.array(humidity)\n",
    "        humidity_numpy = humidity_numpy.astype(float)\n",
    "        pattern = \"\\$(.*?)\\$\"\n",
    "        pressure = re.findall(pattern, string)\n",
    "        pressure_numpy=np.array(pressure)\n",
    "        pressure_numpy = pressure_numpy.astype(float)\n",
    "        pattern = \"\\^(.*?)\\^\"\n",
    "        acceleration = re.findall(pattern, string)\n",
    "        noaccx=[]\n",
    "        accz=[]\n",
    "        accx=[i.split(\"x\", 1)[0] for i in acceleration]\n",
    "        noaccx.append([i.split(\"x\", 1)[1] for i in acceleration])\n",
    "        accy=[i.split(\"x\", 1)[0] for i in noaccx[0]]\n",
    "        accz=[i.split(\"x\", 1)[1] for i in noaccx[0]]\n",
    "        accx_numpy=np.array(accx)\n",
    "        accx_numpy = accx_numpy.astype(float)\n",
    "        accy_numpy=np.array(accy)\n",
    "        accy_numpy = accy_numpy.astype(float)\n",
    "        accz_numpy=np.array(accz)\n",
    "        accz_numpy = accz_numpy.astype(float)\n",
    "        pattern = \"\\@(.*?)\\@\"\n",
    "        gyroscope = re.findall(pattern, string)\n",
    "        nogyrox=[]\n",
    "        gyroz=[]\n",
    "        gyrox=[i.split(\"x\", 1)[0] for i in gyroscope]\n",
    "        nogyrox.append([i.split(\"x\", 1)[1] for i in gyroscope])\n",
    "        gyroy=[i.split(\"x\", 1)[0] for i in nogyrox[0]]\n",
    "        gyroz=[i.split(\"x\", 1)[1] for i in nogyrox[0]]\n",
    "        gyrox_numpy=np.array(gyrox)\n",
    "        gyrox_numpy = gyrox_numpy.astype(float)\n",
    "        gyroy_numpy=np.array(gyroy)\n",
    "        gyroy_numpy = gyroy_numpy.astype(float)\n",
    "        gyroz_numpy=np.array(gyroz)\n",
    "        gyroz_numpy = gyroz_numpy.astype(float)\n",
    "        pattern = \"\\#(.*?)\\#\"\n",
    "        magno = re.findall(pattern, string)\n",
    "        nomagnox=[]\n",
    "        magnoz=[]\n",
    "        magnox=[i.split(\"x\", 1)[0] for i in magno]\n",
    "        nomagnox.append([i.split(\"x\", 1)[1] for i in magno])\n",
    "        magnoy=[i.split(\"x\", 1)[0] for i in nomagnox[0]]\n",
    "        magnoz=[i.split(\"x\", 1)[1] for i in nomagnox[0]]\n",
    "        magnox_numpy=np.array(magnox)\n",
    "        magnox_numpy = magnox_numpy.astype(float)\n",
    "        magnoy_numpy=np.array(magnoy)\n",
    "        magnoy_numpy = magnoy_numpy.astype(float)\n",
    "        magnoz_numpy=np.array(magnoz)\n",
    "        magnoz_numpy = magnoz_numpy.astype(float)\n",
    "        pattern = \"\\*(.*?)\\*\"\n",
    "        distance = re.findall(pattern, string)\n",
    "        distance_numpy=np.array(distance)\n",
    "        distance_numpy = distance_numpy.astype(float)\n",
    "        pattern = \"\\[(.*?)\\)\"\n",
    "        related_timestamps = re.findall(pattern, string)\n",
    "        related_timestamps2=[x for x in related_timestamps if len(x)>100]\n",
    "        related_timestamp=[i.split(\"]\", 1)[0] for i in related_timestamps2]\n",
    "        All_data=pd.DataFrame(related_timestamp,columns=['Timestamps'])\n",
    "        All_data['Temperature'] = temperature_numpy\n",
    "        All_data['Humidity'] = humidity_numpy\n",
    "        All_data['Pressure'] = pressure_numpy\n",
    "        All_data['Distance'] = distance_numpy\n",
    "        All_data['Accx'] = accx_numpy\n",
    "        All_data['Accy'] = accy_numpy\n",
    "        All_data['Accz'] = accz_numpy\n",
    "        All_data['Gyrox'] = gyrox_numpy\n",
    "        All_data['Gyroy'] = gyroy_numpy\n",
    "        All_data['Gyroz'] = gyroz_numpy\n",
    "        All_data['Magnox'] = magnox_numpy\n",
    "        All_data['Magnoy'] = magnoy_numpy\n",
    "        All_data['Magnoz'] = magnoz_numpy\n",
    "        All_data.to_csv(give_output_CSV_folder_path_here+\"\\\\\"+f_name+'_AllSensors.csv',encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}