development/add_string_to_hdf5_dev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "from ramandecompy import dataprep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove('string_test.hdf5')\n",
    "dataprep.new_hdf5('string_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf5 = h5py.File('string_test.hdf5', 'r+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf5['dataset'] = [1, 2, 3, 4, 5, 6, 7]\n",
    "hdf5.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**** string_test.hdf5 ****\n",
      "dataset\n"
     ]
    }
   ],
   "source": [
    "dataprep.view_hdf5('string_test.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataset(h5py_file):\n",
    "    data = (1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 'label')\n",
    "    my_datatype = np.dtype([('fraction', np.float),\n",
    "                        ('center', np.float),\n",
    "                        ('sigma', np.float),\n",
    "                        ('amplitude', np.float),\n",
    "                        ('fwhm', np.float),\n",
    "                        ('height', np.float),\n",
    "                        ('area under the curve', np.float),\n",
    "                        ('label', h5py.special_dtype(vlen=str))])\n",
    "    dataset = h5py_file.create_dataset('label_added', (len(data),), dtype=my_datatype)\n",
    "    data_array = np.array(data, dtype=my_datatype)\n",
    "    dataset = data_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf5 = h5py.File('string_test.hdf5', 'r+')\n",
    "data = (1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 'label')\n",
    "my_datatype = np.dtype([('fraction', np.float),\n",
    "                    ('center', np.float),\n",
    "                    ('sigma', np.float),\n",
    "                    ('amplitude', np.float),\n",
    "                    ('fwhm', np.float),\n",
    "                    ('height', np.float),\n",
    "                    ('area under the curve', np.float),\n",
    "                    ('label', h5py.special_dtype(vlen=str))])\n",
    "dataset = hdf5.create_dataset('label_added', (1,), dtype=my_datatype)\n",
    "data_array = np.array(data, dtype=my_datatype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array((1., 2., 3., 4., 5., 6., 7., 'label'),\n",
       "      dtype=[('fraction', '<f8'), ('center', '<f8'), ('sigma', '<f8'), ('amplitude', '<f8'), ('fwhm', '<f8'), ('height', '<f8'), ('area under the curve', '<f8'), ('label', 'O')])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**** string_test.hdf5 ****\n",
      "dataset\n",
      "label_added\n"
     ]
    }
   ],
   "source": [
    "dataprep.view_hdf5('string_test.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0., 0., 0., 0., 0., 0., 0., '')]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(hdf5['label_added'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[...] = data_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1., 2., 3., 4., 5., 6., 7., 'label')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(hdf5['label_added'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'label'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "foo = list(hdf5['label_added'])\n",
    "foo[0][7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to save strings and floats to the same dataset it was necessary to create a custom dtype which could be applied to a tuple of a known shape, with the last element being a string. Therefore we need to update our existing code so that it expects each peak dataset to contain only a single tuple."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1., 2., 3., 4., 5., 6., 7., 'label')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this can be achived with this line of code\n",
    "foo2 = hdf5['label_added'][:]\n",
    "# since the first and only) element of the dataset is the tuple\n",
    "foo2[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(foo2[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4.0, 5, 6.0, 7]\n",
      "<class 'list'>\n",
      "(1, 2, 3, 4.0, 5, 6.0, 7)\n",
      "<class 'tuple'>\n"
     ]
    }
   ],
   "source": [
    "peak_param = [0]*7\n",
    "peak_param[0] = 1\n",
    "peak_param[1] = 2\n",
    "peak_param[2] = 3\n",
    "peak_param[3] = 4.0\n",
    "peak_param[4] = 5\n",
    "peak_param[5] = 6.0\n",
    "peak_param[6] = 7\n",
    "print(peak_param)\n",
    "print(type(peak_param))\n",
    "\n",
    "peak_tuple = tuple(peak_param)\n",
    "print(peak_tuple)\n",
    "print(type(peak_tuple))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "fraction, sigma, center, amplitude = peak_tuple[0:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "center"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amplitude"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ok so everything looks gucci. super easy to save as tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5, 6, 7]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = [1,2,3,4,5,6,7,'test']\n",
    "test[:7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ok so now a function that will add a label to an `existing peak_dataset`\n",
    "# Brandon --------v  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_label(hdf5_filename, temp, time, peak, label):\n",
    "    \"\"\"\n",
    "    Function that adds a label to a peak dataset in the hdf5 file\n",
    "    \"\"\"\n",
    "    # open hdf5 file as read/write\n",
    "    hdf5 = h5py.File(hdf5_filename, 'r+')\n",
    "    # extract existing data from peak dataset\n",
    "    peak_data = list(hdf5['{}C/{}s/{}'.format(temp, time, peak)])[0]\n",
    "    # make a new tuple that contains the orginal data as well as the label\n",
    "    label_tuple = (label,)\n",
    "    data = tuple(peak_data) +label_tuple\n",
    "    # delete the old dataset so the new one can be saved\n",
    "    del hdf5['{}C/{}s/{}'.format(temp, time, peak)]\n",
    "    # define a custom datatype that allows for a string as the the last tuple element\n",
    "    my_datatype = np.dtype([('fraction', np.float),\n",
    "                        ('center', np.float),\n",
    "                        ('sigma', np.float),\n",
    "                        ('amplitude', np.float),\n",
    "                        ('fwhm', np.float),\n",
    "                        ('height', np.float),\n",
    "                        ('area under the curve', np.float),\n",
    "                        ('label', h5py.special_dtype(vlen=str))])\n",
    "    # recreate the old dataset in the hdf5 file\n",
    "    dataset = hdf5.create_dataset('{}C/{}s/{}'.format(temp, time, peak), (1,), dtype=my_datatype)\n",
    "    # apply custom dtype to data tuple\n",
    "    data_array = np.array(data, dtype=my_datatype)\n",
    "    # write new values to the blank dataset\n",
    "    dataset[...] = data_array\n",
    "    hdf5.close()\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf5_filename = 'spectrafit_dev3.hdf5'\n",
    "temp = 300\n",
    "time = 25\n",
    "peak = 'Peak_01'\n",
    "label = '[Hydrogen]'\n",
    "\n",
    "add_label(hdf5_filename, temp, time, peak, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.66633437e-06, 9.47784158, 314.77, 251.85266517, 18.95568315, 12.48172261, 251.85253919, '[Hydrogen]')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hdf5 = h5py.File(hdf5_filename, 'r+') \n",
    "foo = hdf5['{}C/{}s/{}'.format(temp, time, peak)][0]\n",
    "foo\n",
    "# del hdf5['{}C/{}s/{}'.format(temp, time, peak)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf5.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 2, 3, 4, 5, 6, 7)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = [1,2,3,4,5,6,7,8]\n",
    "foo = [tuple(result[:7]),]\n",
    "foo[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = tuple(result[:7])\n",
    "data_array = np.array(data, dtype='<f8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 2., 3., 4., 5., 6., 7.])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "# from ramandecompy import spectrafit\n",
    "\n",
    "# hdf5_filename = 'spectrafit_dev3.hdf5'\n",
    "# exp_filename = '../ramandecompy/tests/test_files/FA_3.6wt%_300C_55s.csv'\n",
    "\n",
    "\n",
    "# # r+ is read/write mode and will fail if the file does not exist\n",
    "# exp_file = h5py.File(hdf5_filename, 'r+')\n",
    "# if exp_filename.split('.')[-1] == 'xlsx':\n",
    "#     data = pd.read_excel(exp_filename, header=None, names=('wavenumber', 'counts'))\n",
    "# elif exp_filename.split('.')[-1] == 'csv':\n",
    "#     data = pd.read_csv(exp_filename, header=None, names=('wavenumber', 'counts'))\n",
    "# else:\n",
    "#     print('data file type not recognized')\n",
    "# # ensure that the data is listed from smallest wavenumber first\n",
    "# if data['wavenumber'][:1].values > data['wavenumber'][-1:].values:\n",
    "#     data = data.iloc[::-1]\n",
    "#     data.reset_index(inplace=True, drop=True)\n",
    "# else:\n",
    "#     pass\n",
    "# # peak detection and data fitting\n",
    "# fit_result, residuals = spectrafit.fit_data(data['wavenumber'].values, data['counts'].values)\n",
    "# # extract experimental parameters from filename\n",
    "# specs = exp_filename.split('/')[-1].split('.')[:-1]\n",
    "# if len(specs) > 1:\n",
    "#     spec = ''\n",
    "#     for _, element in enumerate(specs):\n",
    "#         spec = str(spec+element)\n",
    "#     specs = spec\n",
    "# specs = specs.split('_')\n",
    "# time = specs[-1]\n",
    "# temp = specs[-2]\n",
    "# # write data to .hdf5\n",
    "# exp_file['{}/{}/wavenumber'.format(temp, time)] = data['wavenumber']\n",
    "# exp_file['{}/{}/counts'.format(temp, time)] = data['counts']\n",
    "# exp_file['{}/{}/residuals'.format(temp, time)] = residuals\n",
    "# for i, result in enumerate(fit_result):\n",
    "#     my_datatype = np.dtype([('fraction', np.float),\n",
    "#                     ('center', np.float),\n",
    "#                     ('sigma', np.float),\n",
    "#                     ('amplitude', np.float),\n",
    "#                     ('fwhm', np.float),\n",
    "#                     ('height', np.float),\n",
    "#                     ('area under the curve', np.float)])\n",
    "#     if i < 9:\n",
    "#         dataset = exp_file.create_dataset('{}/{}/Peak_0{}'.format(temp, time, i+1), (1,), dtype=my_datatype)\n",
    "#     else:\n",
    "#         dataset = exp_file.create_dataset('{}/{}/Peak_{}'.format(temp, time, i+1), (1,), dtype=my_datatype)\n",
    "#     # apply data to tuple\n",
    "#     data = tuple(result[:7])\n",
    "#     data_array = np.array(data, dtype=my_datatype)\n",
    "#     # write new values to the blank dataset\n",
    "#     dataset[...] = data_array\n",
    "# print('Data from {} fit with compound pseudo-Voigt model. Results saved to {}.'.format(exp_filename, hdf5_filename))\n",
    "# exp_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 2., 3., 4., 5., 6., 7.])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}