__ipynb__.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import time \n",
    "import warnings \n",
    "import cProfile \n",
    "import traceback \n",
    "import pstats \n",
    "import os \n",
    "from abc import ABCMeta, abstractmethod\n",
    "from typing import Callable, Generator, Iterable, TYPE_CHECKING, Union, Sized\n",
    "import numpy as np\n",
    "from matplotlib import widgets as wg, pyplot as plt\n",
    "import tempfile \n",
    "import ctypes \n",
    "from numpy.lib import format as fm\n",
    "import sys \n",
    "import inspect \n",
    "import numexpr as ne"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /DataSets/dataSet.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class TrainSets:\n",
    "    EmnistBalanced = \"emnist.balanced.train.112800s.(28,28)i.(47,1)o.zdb\"\n",
    "    Xor = \"xor3.train.8s.(3,1)i.(1,1)o.zdb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class TestSets:\n",
    "    EmnistBalanced = \"emnist.balanced.test.18800s.(28,28)i.(47,1)o.zdb\"\n",
    "    Xor = \"xor3.test.8s.(3,1)i.(1,1)o.zdb\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /Models/model.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/tools/base.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import os\n",
    "pass  # from abc import ABCMeta, abstractmethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np\n",
    "pass  # from matplotlib import pyplot as plt, widgets as wg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseSave(metaclass=ABCMeta):\n",
    "    DEFAULT_DIR: str\n",
    "    DEFAULT_NAME: str\n",
    "    FILE_TYPE: str\n",
    "    @abstractmethod\n",
    "    def saveName(self) -> str:\n",
    "        pass\n",
    "    @abstractmethod\n",
    "    def save(self, file: str = None, replace: bool = False) -> str:\n",
    "        if file is None: file = self.DEFAULT_NAME\n",
    "        if not (fpath := os.path.dirname(file)):\n",
    "            fpath = f\"{os.getcwd()}\\\\{self.DEFAULT_DIR}\\\\\"\n",
    "            fName = file\n",
    "        else:\n",
    "            fpath += '\\\\'\n",
    "            fName = os.path.basename(file)\n",
    "        os.makedirs(fpath, exist_ok=True)\n",
    "        if len(fName) >= (typeLen := 1 + len(self.FILE_TYPE)) and fName[1 - typeLen:] == self.FILE_TYPE:\n",
    "            fName = fName[:-typeLen]\n",
    "            savePath = f\"{fpath}{fName.replace(' ', '_')}\"\n",
    "        else:\n",
    "            savePath = f\"{fpath}{fName.replace(' ', '_')}_{self.saveName().replace(' ', '')}\"\n",
    "        numSavePath = savePath\n",
    "        if not replace:\n",
    "            i = 0\n",
    "            while 1:\n",
    "                if i != 0: numSavePath = f\"{savePath} ({i})\"\n",
    "                if not os.path.exists(f\"{numSavePath}.{self.FILE_TYPE}\"): break\n",
    "                i += 1\n",
    "        dumpFile = f\"{numSavePath}.{self.FILE_TYPE}\"\n",
    "        return dumpFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseLoad(metaclass=ABCMeta):\n",
    "    DEFAULT_DIR: str\n",
    "    FILE_TYPE: str\n",
    "    @classmethod\n",
    "    @abstractmethod\n",
    "    def load(cls, file) -> \"cls\":  # noqa\n",
    "        if file:\n",
    "            if not (fpath := os.path.dirname(file)):\n",
    "                fpath = f\"{os.getcwd()}\\\\{cls.DEFAULT_DIR}\\\\\"\n",
    "                fName = file\n",
    "            else:\n",
    "                fpath += '\\\\'\n",
    "                fName = os.path.basename(file)\n",
    "        else:\n",
    "            raise NameError(\"file not given\")\n",
    "        if '.' not in fName: fName += cls.FILE_TYPE\n",
    "        loadFile = fpath + fName\n",
    "        return loadFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plt.style.use('dark_background')\n",
    "class Plot:  # noqa\n",
    "    @staticmethod\n",
    "    def __init_ax(ax):\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots()\n",
    "        else:\n",
    "            ax, fig = ax, ax.figure\n",
    "        return ax\n",
    "    @staticmethod\n",
    "    def __plotMulti(count, plotter, rows, columns):\n",
    "        MAX = rows * columns\n",
    "        fig = plt.figure()\n",
    "        axes = [fig.add_subplot(rows + 1, columns, i + 1) for i in range(MAX)]\n",
    "        butPrev = wg.Button(ax := fig.add_subplot(rows + 1, 2, 2 * rows + 1), '<-', color='red', hovercolor='blue')\n",
    "        ax.set_aspect(.1)\n",
    "        butNext = wg.Button(ax := fig.add_subplot(rows + 1, 2, 2 * rows + 2), '->', color='red', hovercolor='blue')\n",
    "        ax.set_aspect(.1)\n",
    "        fig.page = 0\n",
    "        def onclick(_page):\n",
    "            if _page == 0:\n",
    "                butPrev.active = False\n",
    "                butPrev.ax.patch.set_visible(False)\n",
    "            else:\n",
    "                butPrev.active = True\n",
    "                butPrev.ax.patch.set_visible(True)\n",
    "            if _page == count // MAX:\n",
    "                butNext.active = False\n",
    "                butNext.ax.patch.set_visible(False)\n",
    "            else:\n",
    "                butNext.active = True\n",
    "                butNext.ax.patch.set_visible(True)\n",
    "            fig.page = _page\n",
    "            [(_ax.clear(), _ax.set_xticks([]), _ax.set_yticks([])) for _ax in axes]\n",
    "            plotter(_page, axes)\n",
    "            fig.subplots_adjust(.01, .01, .99, .99, 0, 0)\n",
    "            fig.canvas.draw()\n",
    "        butNext.on_clicked(lambda *_: onclick(fig.page + 1))\n",
    "        butPrev.on_clicked(lambda *_: onclick(fig.page - 1))\n",
    "        onclick(fig.page)\n",
    "        return axes\n",
    "    @staticmethod\n",
    "    def plotHeight(xs, ys=None, cluster=False, join=True,\n",
    "                   scatter=False, scatterLabels=None, scatterRotation=0, scatterSize=100,\n",
    "                   text='', textPos=(.01, .01), textC='yellow', ax=None, multi=False, rows=4, columns=4):\n",
    "        if multi:\n",
    "            MAX = rows * columns\n",
    "            length = np.shape(xs)[0]\n",
    "            if text is None:\n",
    "                text = range(length)\n",
    "            if scatterLabels is None:\n",
    "                scatterLabels = range(length)\n",
    "            if ys is None:\n",
    "                ys = [None for _ in range(length)]\n",
    "            def plotter(_page, axes):\n",
    "                if (to := (_page + 1) * MAX) > length:\n",
    "                    to = length\n",
    "                for ind, x in enumerate(xs[_page * MAX:to]):\n",
    "                    Plot.plotHeight(x, ys[ind], cluster, join, scatter, scatterLabels, scatterRotation,\n",
    "                                    text=str(text[_page * MAX + ind]), ax=axes[ind])\n",
    "            return Plot.__plotMulti(xs.shape[0], plotter, rows, columns)\n",
    "        ax = Plot.__init_ax(ax)\n",
    "        args = (xs, ys)\n",
    "        if ys is None:\n",
    "            args = (xs,)\n",
    "        if not cluster:\n",
    "            args = [[arg] for arg in args]\n",
    "            scatterLabels = [scatterLabels]\n",
    "        for i in range(len(xs)):\n",
    "            points = [arg[i] for arg in args if arg[i] is not None]\n",
    "            if join:\n",
    "                ax.plot(*points, c=np.random.rand(3))\n",
    "            if scatter or scatterLabels is not None:\n",
    "                ax.scatter(*points, s=scatterSize, color=\"gray\")\n",
    "                if scatterLabels is not None:\n",
    "                    for *point, label in zip(*points, scatterLabels[i]):\n",
    "                        ax.annotate(label, point, rotation=scatterRotation)\n",
    "        ax.text(*textPos, text, transform=ax.transAxes, c=textC)\n",
    "        return ax\n",
    "    @staticmethod\n",
    "    def plotMap(vect, text=None, textPos=(.01, .01), textC='yellow', ax=None, rows=4, columns=4):\n",
    "        if len(np.shape(vect)) != 2:\n",
    "            MAX = rows * columns\n",
    "            length = np.shape(vect)[0]\n",
    "            if text is None:\n",
    "                text = range(length)\n",
    "            def plotter(_page, axes):\n",
    "                if (to := (_page + 1) * MAX) > length:\n",
    "                    to = length\n",
    "                for i, im in enumerate(vect[_page * MAX:to]):\n",
    "                    Plot.plotMap(im, text=str(text[_page * MAX + i]), ax=axes[i])\n",
    "            return Plot.__plotMulti(length, plotter, rows, columns)\n",
    "        ax = Plot.__init_ax(ax)\n",
    "        ax.imshow(vect)\n",
    "        ax.text(*textPos, text, transform=ax.transAxes, c=textC)\n",
    "        return ax\n",
    "    @staticmethod\n",
    "    def show():\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/tools/helperClass.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import tempfile\n",
    "pass  # import ctypes\n",
    "pass  # from typing import TYPE_CHECKING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if TYPE_CHECKING:\n",
    "    pass  # from ..tools import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np\n",
    "pass  # from numpy.lib import format as fm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class NumpyDataCache(np.ndarray):\n",
    "    def __new__(cls, array):\n",
    "        return cls.writeNpyCache(array)\n",
    "    @staticmethod\n",
    "    def writeNpyCache(array: \"np.ndarray\") -> np.ndarray:\n",
    "        with tempfile.NamedTemporaryFile(suffix='.npy') as file:\n",
    "            np.save(file, array)\n",
    "            file.seek(0)\n",
    "            fm.read_magic(file)\n",
    "            fm.read_array_header_1_0(file)\n",
    "            memMap = np.memmap(file, mode='r', shape=array.shape, dtype=array.dtype, offset=file.tell())\n",
    "        return memMap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Collections:\n",
    "    def __repr__(self):\n",
    "        return f\"<{self.__class__.__name__}:{self.collectables}>\"\n",
    "\n",
    "    # todo: make collectables Type[_<class>] itself, and/or create Collection class generator in general\n",
    "    def __init__(self, *collectables):\n",
    "        self.collectables = collectables\n",
    "    def __call__(self, length):\n",
    "        return self.get(length)\n",
    "    def get(self, length):\n",
    "        trueCollectables = []\n",
    "        prevCollectable = None\n",
    "        numEllipsis = self.collectables.count(Ellipsis)\n",
    "        numCollectables = len(self.collectables) - numEllipsis\n",
    "        vacancy = length - numCollectables\n",
    "        for collectable in self.collectables:\n",
    "            if collectable == Ellipsis:\n",
    "                for i in range(filled := (vacancy // numEllipsis)):\n",
    "                    trueCollectables.append(prevCollectable)\n",
    "                vacancy -= filled\n",
    "                numEllipsis -= 1\n",
    "                continue\n",
    "            trueCollectables.append(collectable)\n",
    "            prevCollectable = collectable\n",
    "        return trueCollectables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    kernel32 = ctypes.windll.kernel32\n",
    "    kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)\n",
    "except:  # noqa\n",
    "    pass\n",
    "# noinspection SpellCheckingInspection\n",
    "class PrintCols:  # noqa\n",
    "    CEND = '\\33[0m'\n",
    "    CBOLD = '\\33[1m'\n",
    "    CITALIC = '\\33[3m'\n",
    "    CURL = '\\33[4m'\n",
    "    CBLINK = '\\33[5m'\n",
    "    CBLINK2 = '\\33[6m'\n",
    "    CSELECTED = '\\33[7m'\n",
    "    CBOLDITALIC = CBOLD + CITALIC\n",
    "    CURLBOLD = CBOLD + CURL\n",
    "    CITALICURL = CITALIC + CURL\n",
    "    CBOLDITALICURL = CBOLD + CITALIC + CURL\n",
    "    CBLACK = '\\33[30m'\n",
    "    CRED = '\\33[31m'\n",
    "    CGREEN = '\\33[32m'\n",
    "    CYELLOW = '\\33[33m'\n",
    "    CBLUE = '\\33[34m'\n",
    "    CVIOLET = '\\33[35m'\n",
    "    CBEIGE = '\\33[36m'\n",
    "    CWHITE = '\\33[37m'\n",
    "    CBLACKBG = '\\33[40m'\n",
    "    CREDBG = '\\33[41m'\n",
    "    CGREENBG = '\\33[42m'\n",
    "    CYELLOWBG = '\\33[43m'\n",
    "    CBLUEBG = '\\33[44m'\n",
    "    CVIOLETBG = '\\33[45m'\n",
    "    CBEIGEBG = '\\33[46m'\n",
    "    CWHITEBG = '\\33[47m'\n",
    "    CGREY = '\\33[90m'\n",
    "    CRED2 = '\\33[91m'\n",
    "    CGREEN2 = '\\33[92m'\n",
    "    CYELLOW2 = '\\33[93m'\n",
    "    CBLUE2 = '\\33[94m'\n",
    "    CVIOLET2 = '\\33[95m'\n",
    "    CBEIGE2 = '\\33[96m'\n",
    "    CWHITE2 = '\\33[97m'\n",
    "    CGREYBG = '\\33[100m'\n",
    "    CREDBG2 = '\\33[101m'\n",
    "    CGREENBG2 = '\\33[102m'\n",
    "    CYELLOWBG2 = '\\33[103m'\n",
    "    CBLUEBG2 = '\\33[104m'\n",
    "    CVIOLETBG2 = '\\33[105m'\n",
    "    CBEIGEBG2 = '\\33[106m'\n",
    "    CWHITEBG2 = '\\33[107m'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/tools/magicProperty.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class MagicBase:\n",
    "    def __new__(cls, *args, **kwargs):\n",
    "        obj = super(MagicBase, cls).__new__(cls)\n",
    "        obj.toMagicProperty = set()\n",
    "        return obj\n",
    "    def __magic_start__(self):\n",
    "        self.toMagicProperty = set(self.__dict__.keys())\n",
    "    def __magic_end__(self):\n",
    "        self.toMagicProperty = set(self.__dict__.keys()) - self.toMagicProperty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class MagicProperty(property):\n",
    "    def __init__(self, fget=None, fset=None, fdel=None, doc=None):\n",
    "        super(MagicProperty, self).__init__(fget, self.makeMagicF(fset), self.makeMagicF(fdel), doc)\n",
    "        self.__obj = self.getCaller()\n",
    "    def makeMagicF(self, _f):\n",
    "        def f(*args, **kwargs):\n",
    "            if self.__magic__():\n",
    "                return _f(*args, **kwargs)\n",
    "            else:\n",
    "                raise AttributeError(\"Attribute is read only\")\n",
    "        return f\n",
    "    def __magic__(self, stack=1):\n",
    "        caller = self.getCaller(stack + 1)\n",
    "        return any(c1 == c2 and c1 is not None for c1, c2 in zip(caller, self.__obj)) or \\\n",
    "            (any(self.__obj[2] == base.__name__ for base in caller[1].__bases__)\n",
    "                if self.__obj[:2] == (None, None) and caller[1] is not None else 0)\n",
    "    @staticmethod\n",
    "    def getCaller(stack=1):\n",
    "        caller = (callStack := inspect.stack()[stack + 1][0].f_locals).get('self')\n",
    "        _return = caller, caller.__class__, caller.__class__.__name__\n",
    "        if caller is None:\n",
    "            _return = None, (caller := callStack.get('cls')), caller.__name__ if caller is not None else None\n",
    "        if caller is None: _return = None, None, callStack.get('__qualname__')\n",
    "        return _return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def makeMetaMagicProperty(*inherits):\n",
    "    class MetaProperty(*inherits, type):  # todo: improve implementation method\n",
    "        def __call__(cls, *args, **kwargs):\n",
    "            __obj = super(MetaProperty, cls).__call__(*args, **kwargs)\n",
    "            __dict__ = {}\n",
    "            for key, val in __obj.__dict__.items():\n",
    "                if key.isupper():\n",
    "                    __dict__[(_name := '__magic' + key)] = val\n",
    "                    setattr(cls, key, MagicProperty(lambda self, _name=_name: getattr(self, _name),\n",
    "                                                    lambda self, _val, _name=_name: setattr(self, _name, _val)))\n",
    "            __obj.__dict__.update(__dict__)\n",
    "            return __obj\n",
    "    return MetaProperty"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/Topologies/activationFunction.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from typing import Union\n",
    "pass  # from abc import ABCMeta, abstractmethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseActivationFunction(metaclass=ABCMeta):\n",
    "    ONE = np.float32(1)\n",
    "    E = np.float32(np.e)\n",
    "    def __repr__(self):\n",
    "        return f\"<{self.__class__.__name__}>\"\n",
    "    @abstractmethod\n",
    "    def activation(self, x: np.ndarray) -> \"np.ndarray\":\n",
    "        pass\n",
    "    @abstractmethod\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> \"np.ndarray\":\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Sigmoid(BaseActivationFunction):\n",
    "    def __repr__(self):\n",
    "        smooth = self.SMOOTH\n",
    "        offset = self.OFFSET\n",
    "        return f\"{super(Sigmoid, self).__repr__()[:-1]}: {smooth=}: {offset=}>\"\n",
    "    def __init__(self, smooth: Union[int, float] = 1, offset: Union[int, float] = 0):\n",
    "        self.SMOOTH = np.float32(smooth)\n",
    "        self.OFFSET = np.float32(offset)\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        return self.ONE / (self.ONE + self.E ** (-self.SMOOTH * (x - self.OFFSET)))\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> np.ndarray:\n",
    "        return self.SMOOTH * (activatedX * (self.ONE - activatedX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class TanH(BaseActivationFunction):\n",
    "    def __repr__(self):\n",
    "        alpha = self.ALPHA\n",
    "        return f\"{super(TanH, self).__repr__()[:-1]}: {alpha=}>\"\n",
    "    def __init__(self, alpha: Union[int, float] = 1):\n",
    "        self.ALPHA = np.float32(alpha)\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        return np.arctan(self.ALPHA * x)\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> np.ndarray:\n",
    "        return self.ALPHA * np.square(np.cos(activatedX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Relu(BaseActivationFunction):\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        return x * (x > 0)\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> np.ndarray:\n",
    "        return self.ONE * (activatedX != 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class PRelu(BaseActivationFunction):\n",
    "    def __repr__(self):\n",
    "        leak = self.LEAK\n",
    "        return f\"{super(PRelu, self).__repr__()[:-1]}: {leak=}>\"\n",
    "    def __init__(self, leak: Union[int, float] = 0.01):\n",
    "        if leak < 0: raise ValueError(\"parameter 'leak' cannot be less than zero\")\n",
    "        self.LEAK = np.float32(leak)\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        return np.where(x > 0, x, self.LEAK * x)\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> np.ndarray:\n",
    "        return np.where(activatedX <= 0, self.LEAK, self.ONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Elu(BaseActivationFunction):\n",
    "    def __repr__(self):\n",
    "        alpha = self.ALPHA\n",
    "        return f\"{super(Elu, self).__repr__()[:-1]}: {alpha=}>\"\n",
    "    def __init__(self, alpha: Union[int, float] = 1):\n",
    "        if alpha < 0: raise ValueError(\"parameter 'alpha' cannot be less than zero\")\n",
    "        self.ALPHA = np.float32(alpha)\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        return np.where(x > 0, x, self.ALPHA * (self.E ** x - 1))\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> np.ndarray:\n",
    "        return np.where(activatedX <= 0, activatedX + self.ALPHA, self.ONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class SoftMax(BaseActivationFunction):\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        numerator = self.E ** (x - x.max(axis=-2, keepdims=True))\n",
    "        return numerator / numerator.sum(axis=-2, keepdims=1)\n",
    "    def activatedDerivative(self, activatedX: np.ndarray):\n",
    "        jacobian = np.einsum('...ij,...kj->...jik', activatedX, activatedX, optimize='greedy')\n",
    "        diagIndexes = np.diag_indices(jacobian.shape[-1])\n",
    "        jacobian[..., diagIndexes[0], diagIndexes[1]] = \\\n",
    "            (activatedX * (1 - activatedX)).transpose().reshape(jacobian.shape[:-1])\n",
    "        return jacobian.sum(axis=-1).transpose().reshape(activatedX.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class SoftPlus(BaseActivationFunction):\n",
    "    def activation(self, x: np.ndarray) -> np.ndarray:\n",
    "        return np.log(self.ONE + self.E ** x)\n",
    "    def activatedDerivative(self, activatedX: np.ndarray) -> np.ndarray:\n",
    "        return self.ONE - self.E ** -activatedX"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/Topologies/initializer.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from abc import ABCMeta, abstractmethod\n",
    "pass  # from typing import TYPE_CHECKING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if TYPE_CHECKING:\n",
    "    pass  # from ..NeuralNetworks import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseInitializer(metaclass=ABCMeta):\n",
    "    def __repr__(self):\n",
    "        return f\"<{self.__class__.__name__}>\"\n",
    "    @abstractmethod\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.rnd = np.random.default_rng()\n",
    "    def __call__(self, shape: \"Base.Shape\") -> \"np.ndarray\":\n",
    "        return self._initialize(shape)\n",
    "    @abstractmethod\n",
    "    def _initialize(self, shape: \"Base.Shape\") -> \"np.ndarray\":\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Uniform(BaseInitializer):\n",
    "    def __repr__(self):\n",
    "        start = self.start\n",
    "        stop = self.stop\n",
    "        return f\"{super(Uniform, self).__repr__()[:-1]}: {start=}: {stop}>\"\n",
    "    def __init__(self, start: \"float\" = -1, stop: \"float\" = 1):\n",
    "        super(Uniform, self).__init__()\n",
    "        self.start = start\n",
    "        self.stop = stop\n",
    "    def _initialize(self, shape: \"Base.Shape\") -> \"np.ndarray\":\n",
    "        return self.rnd.uniform(self.start, self.stop, shape.HIDDEN).astype(dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Normal(BaseInitializer):\n",
    "    def __repr__(self):\n",
    "        scale = self.scale\n",
    "        return f\"{super(Normal, self).__repr__()[:-1]}: {scale=}>\"\n",
    "    def __init__(self, scale: \"float\" = 1):\n",
    "        super(Normal, self).__init__()\n",
    "        self.scale = scale\n",
    "    def _initialize(self, shape: \"Base.Shape\") -> \"np.ndarray\":\n",
    "        return self.rnd.standard_normal(shape.HIDDEN, dtype=np.float32) * self.scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Xavier(BaseInitializer):\n",
    "    def __repr__(self):\n",
    "        he = self.he\n",
    "        return f\"{super(Xavier, self).__repr__()[:-1]}: {he=}>\"\n",
    "    def __init__(self, he: \"float\" = 1):\n",
    "        super(Xavier, self).__init__()\n",
    "        self.he = he\n",
    "    def _initialize(self, shape: \"Base.Shape\") -> \"np.ndarray\":\n",
    "        return self.rnd.standard_normal(shape.HIDDEN, dtype=np.float32) * (self.he / np.prod(shape.INPUT)) ** 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class NormalizedXavier(BaseInitializer):\n",
    "    def __repr__(self):\n",
    "        he = self.he\n",
    "        return f\"{super(NormalizedXavier, self).__repr__()[:-1]}: {he=}>\"\n",
    "    def __init__(self, he: \"float\" = 6):\n",
    "        super(NormalizedXavier, self).__init__()\n",
    "        self.he = he\n",
    "    def _initialize(self, shape: \"Base.Shape\"):\n",
    "        return self.rnd.standard_normal(shape.HIDDEN, dtype=np.float32) * (\n",
    "                self.he / (np.prod(shape.INPUT) + np.prod(shape.OUTPUT)) ** (\n",
    "                2 / (len(shape.INPUT) + len(shape.OUTPUT)))) ** 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/Topologies/lossFunction.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from abc import ABCMeta, abstractmethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseLossFunction(metaclass=ABCMeta):\n",
    "    def __repr__(self):\n",
    "        return f\"{self.__class__.__name__}\"\n",
    "    def __call__(self, output, target):\n",
    "        return self._eval(output, target)\n",
    "    @abstractmethod\n",
    "    def _eval(self, output, target):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class MeanSquare(BaseLossFunction):\n",
    "    def _eval(self, output, target):\n",
    "        delta = output - target\n",
    "        return (delta * delta).sum(axis=1).mean(), delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class CrossEntropy(BaseLossFunction):\n",
    "    def _eval(self, output, target):\n",
    "        cross = -np.log(output) * target\n",
    "        return (cross * cross).sum(axis=1).mean(), output - target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/Topologies/optimizer.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from typing import Callable\n",
    "pass  # from abc import ABCMeta, abstractmethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np\n",
    "pass  # import numexpr as ne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseOptimizer(metaclass=ABCMeta):\n",
    "    __args, __kwargs = (), {}\n",
    "    ZERO, ONE = np.float32(0), np.float32(1)\n",
    "    def __repr__(self):\n",
    "        lr = self.LEARNING_RATE\n",
    "        return f\"<{self.__class__.__name__}:{lr=}>\"\n",
    "    def __new__(cls, *args, **kwargs):\n",
    "        cls.__args, cls.__kwargs = args, kwargs\n",
    "        obj = super(BaseOptimizer, cls).__new__(cls)\n",
    "        obj.__init__(*args, **kwargs)\n",
    "        return obj\n",
    "    @classmethod\n",
    "    def __new_copy__(cls):\n",
    "        return cls.__new__(cls, *cls.__args, *cls.__kwargs)\n",
    "    def __init__(self, learningRate: float):\n",
    "        self.LEARNING_RATE = np.float32(learningRate)\n",
    "    def __call__(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\"):\n",
    "        return self._optimize(grad, theta)\n",
    "    @abstractmethod\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class GradientDecent(BaseOptimizer):\n",
    "    def __init__(self, learningRate: float = None):\n",
    "        if learningRate is None: learningRate = .001\n",
    "        super(GradientDecent, self).__init__(learningRate)\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        delta = grad(theta)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        return ne.evaluate(\"delta * LEARNING_RATE\", local_dict=local_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Decay(BaseOptimizer):\n",
    "    def __repr__(self):\n",
    "        decay = self.DECAY\n",
    "        return f\"{super(Decay, self).__repr__()[:-1]}: {decay=}>\"\n",
    "    def __init__(self, learningRate: float = None, decay: float = None):\n",
    "        if learningRate is None: learningRate = .001\n",
    "        super(Decay, self).__init__(learningRate)\n",
    "        if decay is None: decay = self.LEARNING_RATE ** 2\n",
    "        self.DECAY = np.float32(decay)\n",
    "        self.decayCounter = self.ZERO\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        delta = grad(theta)\n",
    "        self.decayCounter += self.ONE\n",
    "        locals()['ONE'] = self.ONE\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        return ne.evaluate(\"delta * LEARNING_RATE / (ONE + decayCounter * DECAY)\", local_dict=local_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Momentum(BaseOptimizer):\n",
    "    def __repr__(self):\n",
    "        moment = self.MOMENT\n",
    "        return f\"{super(Momentum, self).__repr__()[:-1]}: {moment=}>\"\n",
    "    def __init__(self, learningRate: float = None, moment: float = None):\n",
    "        if learningRate is None: learningRate = .001\n",
    "        super(Momentum, self).__init__(learningRate)\n",
    "        if moment is None: moment = .5\n",
    "        self.MOMENT = np.float32(moment)\n",
    "        self.prevDelta = self.ZERO\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        delta = grad(theta)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        self.prevDelta = momentDelta = ne.evaluate(\"LEARNING_RATE * delta + MOMENT * prevDelta\", local_dict=local_dict)\n",
    "        return momentDelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class NesterovMomentum(BaseOptimizer):\n",
    "    def __repr__(self):\n",
    "        moment = self.MOMENT\n",
    "        return f\"{super(NesterovMomentum, self).__repr__()[:-1]}: {moment=}>\"\n",
    "    def __init__(self, learningRate: float = None, moment: float = None):\n",
    "        if learningRate is None: learningRate = .001\n",
    "        super(NesterovMomentum, self).__init__(learningRate)\n",
    "        if moment is None: moment = .5\n",
    "        self.MOMENT = np.float32(moment)\n",
    "        self.prevDelta = self.ZERO\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        delta = grad(theta - self.MOMENT * self.prevDelta)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        self.prevDelta = momentDelta = ne.evaluate(\"LEARNING_RATE * delta + MOMENT * prevDelta\", local_dict=local_dict)\n",
    "        return momentDelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class AdaGrad(BaseOptimizer):\n",
    "    def __repr__(self):\n",
    "        eps = self.EPSILON\n",
    "        return f\"{super(AdaGrad, self).__repr__()[:-1]}: {eps=}>\"\n",
    "    def __init__(self, learningRate: float = None, epsilon: float = None):\n",
    "        if learningRate is None: learningRate = .01\n",
    "        super(AdaGrad, self).__init__(learningRate)\n",
    "        if epsilon is None: epsilon = 1e-7\n",
    "        self.EPSILON = np.float32(epsilon)\n",
    "        self.summationSquareDelta = self.ZERO\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        delta = grad(theta)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        self.summationSquareDelta = ne.evaluate('summationSquareDelta + delta * delta', local_dict=local_dict)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        return ne.evaluate('delta * LEARNING_RATE / sqrt(summationSquareDelta + EPSILON)', global_dict=local_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class RmsProp:\n",
    "    def __init__(self):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class AdaDelta:\n",
    "    def __init__(self):\n",
    "        raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Adam(BaseOptimizer):\n",
    "    def __repr__(self):\n",
    "        b1, b2 = self.BETA1, self.BETA2\n",
    "        eps = self.EPSILON\n",
    "        return f\"{super(Adam, self).__repr__()[:-1]}: {b1=}: {b2=}: {eps=}>\"\n",
    "    def __init__(self, learningRate: float = None, beta1: float = None, beta2: float = None, epsilon: float = None,\n",
    "                 decay: float = None):\n",
    "        if learningRate is None: learningRate = .001\n",
    "        super(Adam, self).__init__(learningRate)\n",
    "        if beta1 is None: beta1 = .9\n",
    "        self.BETA1 = np.float32(beta1)\n",
    "        self.BETA1_BAR = 1 - self.BETA1\n",
    "        if beta2 is None: beta2 = .999\n",
    "        self.BETA2 = np.float32(beta2)\n",
    "        self.BETA2_BAR = 1 - self.BETA2\n",
    "        if epsilon is None: epsilon = 1e-7\n",
    "        self.EPSILON = np.float32(epsilon)\n",
    "        if decay is None: decay = NotImplemented  # todo: implement decay on decayCounter?\n",
    "        # self.DECAY = np.float32(decay)\n",
    "        self.decayCounter = self.ONE\n",
    "        self.weightedSummationDelta = self.ZERO\n",
    "        self.weightedSummationSquareDelta = self.ZERO\n",
    "    def _optimize(self, grad: Callable[[\"np.ndarray\"], \"np.ndarray\"], theta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        delta = grad(theta)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        self.weightedSummationDelta = ne.evaluate(\n",
    "            \"BETA1 * weightedSummationDelta + BETA1_BAR * delta\", local_dict=local_dict)\n",
    "        self.weightedSummationSquareDelta = ne.evaluate(\n",
    "            \"BETA2 * weightedSummationSquareDelta + BETA2_BAR * delta * delta\", local_dict=local_dict)\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        weightedSummationDeltaHat = ne.evaluate(\n",
    "            \"weightedSummationDelta / (1 - BETA1 ** decayCounter)\", local_dict=local_dict)\n",
    "        weightedSummationSquareDeltaHat = ne.evaluate(\n",
    "            \"weightedSummationSquareDelta / (1 - BETA2 ** decayCounter)\", local_dict=local_dict)\n",
    "        self.decayCounter += self.ONE\n",
    "        (local_dict := vars(self)).update(locals())\n",
    "        return ne.evaluate(\n",
    "            \"LEARNING_RATE * weightedSummationDeltaHat / sqrt(weightedSummationSquareDeltaHat + EPSILON)\",\n",
    "            local_dict=local_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/tools/helperFunction.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import sys\n",
    "pass  # import time\n",
    "pass  # from typing import TYPE_CHECKING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if TYPE_CHECKING:\n",
    "    pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from .helperClass import PrintCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def copyNumpyList(lis: list[\"np.ndarray\"]):\n",
    "    copyList = []\n",
    "    for array in lis: copyList.append(array.copy())\n",
    "    return copyList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def iterable(var):\n",
    "    try:\n",
    "        iter(var)\n",
    "        return True\n",
    "    except TypeError:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def secToHMS(seconds, hms=('h', 'm', 's')):\n",
    "    encode = f'%S{hms[2]}'\n",
    "    if (tim := time.gmtime(seconds)).tm_min != 0: encode = f'%M{hms[1]}' + encode\n",
    "    if tim.tm_hour != 0: encode = f'%H{hms[0]}' + encode\n",
    "    return time.strftime(encode, tim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def statPrinter(key, value, *, prefix='', suffix=PrintCols.CEND, end=' '):\n",
    "    print(prefix + f\"{key}:{value}\" + suffix, end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def getSize(obj, seen=None, ref=''):\n",
    "    \"\"\"Recursively finds size of objects\"\"\"\n",
    "    size = sys.getsizeof(obj)\n",
    "    if seen is None: seen = set()\n",
    "    if (obj_id := id(obj)) in seen: return 0\n",
    "    # Important mark as seen *before* entering recursion to gracefully handle\n",
    "    # self-referential objects\n",
    "    seen.add(obj_id)\n",
    "    ref += str(obj.__class__)\n",
    "    if isinstance(obj, dict):\n",
    "        size += sum([getSize(obj[k], seen, ref + str(k)) for k in obj.keys()])\n",
    "        size += sum([getSize(k, seen, ref) for k in obj.keys()])\n",
    "    elif hasattr(obj, '__dict__'):\n",
    "        size += getSize(obj.__dict__, seen, ref)\n",
    "    elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)):\n",
    "        size += sum([getSize(i, seen, ref) for i in obj])\n",
    "    if size > 1024 * 10:  # show files > 10Mb\n",
    "        print(obj.__class__, size)\n",
    "        print(ref, '\\n')\n",
    "    return size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/tools/__init__.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from .base import BaseSave, BaseLoad, Plot\n",
    "pass  # from .helperClass import NumpyDataCache, Collections, PrintCols\n",
    "pass  # from .helperFunction import copyNumpyList, iterable, secToHMS, statPrinter, getSize\n",
    "pass  # from .magicProperty import MagicBase, MagicProperty, makeMetaMagicProperty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "__all__ = [\n",
    "    \"BaseSave\", \"BaseLoad\", \"Plot\",\n",
    "    \"NumpyDataCache\", \"Collections\", \"PrintCols\", \"copyNumpyList\", \"iterable\", \"secToHMS\", \"statPrinter\", \"getSize\",\n",
    "    \"MagicBase\", \"MagicProperty\", \"makeMetaMagicProperty\",\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/Topologies/dataBase.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import warnings\n",
    "pass  # from typing import Iterable, Sized, Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np\n",
    "pass  # import numexpr as ne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from ..tools import NumpyDataCache, BaseSave, BaseLoad, Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DataBase(BaseSave, BaseLoad):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    DEFAULT_DIR = 'DataSets'\n",
    "    DEFAULT_NAME = 'db'\n",
    "    FILE_TYPE = '.zdb'\n",
    "    LARGE_VAL = 5\n",
    "    def __repr__(self):\n",
    "        Shape = {'SIZE': self.size, 'BatchSize': self.batchSize, 'INPUT': self.inpShape, 'TARGET': self.tarShape}\n",
    "        return f\"<{self.__class__.__name__}:{self.NAME}:{Shape=}>\"\n",
    "    def __str__(self):\n",
    "        HotEncode = {'INPUT': self.hotEncodeInp, 'TARGET': self.hotEncodeTar}\n",
    "        SetDType = {'INPUT': self.inputSetDType, 'TARGET': self.targetSetDType}\n",
    "        NormFactor = {'INPUT': f\"Max:{(Max := self.inputMax)}, \"\n",
    "                               f\"Norm:{(Norm := self.inputSetNormFactor)}, {Max*Norm=}\",\n",
    "                      'TARGET': f\"Max:{(Max := self.targetMax)}, \"\n",
    "                                f\"Norm:{(Norm := self.targetSetNormFactor)}, {Max*Norm=}\"}\n",
    "        return f\"{self.__repr__()[:-1]}:\\n\\t{HotEncode=}\\n\\t{SetDType=}\\n\\t{NormFactor=}>\"\n",
    "    def saveName(self) -> str:\n",
    "        return f\"{self.size}s_{self.inpShape}i_{self.tarShape}o\"\n",
    "    def save(self, file: str = None, replace: bool = False) -> str:\n",
    "        dumpFile = super(DataBase, self).save(file, replace)\n",
    "        saveInputSet = self.inputs * self.inputSetNormFactor\n",
    "        if self.hotEncodeInp: saveInputSet = self.oneHotDecode(saveInputSet)\n",
    "        saveTargetSet = self.targets * self.targetSetNormFactor\n",
    "        if self.hotEncodeTar: saveTargetSet = self.oneHotDecode(saveTargetSet)\n",
    "        np.savez_compressed(dumpFile, inputSet=saveInputSet.astype(self.inputSetDType),\n",
    "                            targetSet=saveTargetSet.astype(self.targetSetDType))\n",
    "        return dumpFile\n",
    "    @classmethod\n",
    "    def load(cls, file: str, *DataBase_args, **DataBase_kwargs) -> \"DataBase\":\n",
    "        f\"\"\"\n",
    "        :param file: path like or name\n",
    "        :param DataBase_args: to {DataBase.__init__}(normalizeInp, normalizeTar, reshapeInp, reshapeTar,\n",
    "        oneHotMaxInp, oneHotMaxTar, name)\n",
    "        \"\"\"\n",
    "        loadFile = super(DataBase, cls).load(file)\n",
    "        nnLoader = np.load(loadFile, mmap_mode='r')\n",
    "        inputSet, targetSet = nnLoader['inputSet'], nnLoader['targetSet']\n",
    "        return DataBase(inputSet, targetSet, *DataBase_args, **DataBase_kwargs)\n",
    "    def __getitem__(self, item):\n",
    "        return self.inputs[(i := self.indices[item])], self.targets[i]\n",
    "    def __init__(self,\n",
    "                 inputSet: Iterable and Sized, targetSet: Iterable and Sized,\n",
    "                 normalizeInp: float = None, normalizeTar: float = None,\n",
    "                 reshapeInp=None, reshapeTar=None,\n",
    "                 oneHotMaxInp=None, oneHotMaxTar=None,\n",
    "                 name: str = ''):\n",
    "        if (size := len(inputSet)) != len(targetSet): raise Exception(\"Both input and target set must be of same size\")\n",
    "        self.NAME = name\n",
    "        self.inputSetDType, self.targetSetDType = inputSet.dtype, targetSet.dtype\n",
    "        self.hotEncodeInp = self.hotEncodeTar = False\n",
    "        if len(np.shape(inputSet)) == 1: inputSet, self.hotEncodeInp = self.oneHotEncode(inputSet, oneHotMaxInp)\n",
    "        if len(np.shape(targetSet)) == 1: targetSet, self.hotEncodeTar = self.oneHotEncode(targetSet, oneHotMaxTar)\n",
    "        if (maxI := np.max(inputSet)) >= self.LARGE_VAL and normalizeInp is None and not self.hotEncodeInp:\n",
    "            warnings.showwarning(f\"inputSet has element(s) with values till {maxI} which may cause nan training, \"\n",
    "                                 f\"use of param 'normalizeInp=<max>' is recommended\", FutureWarning, 'dataBase.py', 0)\n",
    "        if (maxT := np.max(targetSet)) >= self.LARGE_VAL and normalizeTar is None and not self.hotEncodeTar:\n",
    "            warnings.showwarning(f\"targetSet has element(s) with values till {maxT} which may cause nan training, \"\n",
    "                                 f\"use of param 'normalizeTar=<max>' is recommended\", FutureWarning, 'dataBase.py', 0)\n",
    "        inputSet, self.inputSetNormFactor = self.normalize(np.array(inputSet, dtype=np.float32), normalizeInp)\n",
    "        targetSet, self.targetSetNormFactor = self.normalize(np.array(targetSet, dtype=np.float32), normalizeTar)\n",
    "        self.inputMax, self.targetMax = inputSet.max(), targetSet.max()\n",
    "        if reshapeInp is not None: inputSet = inputSet.reshape((size, *reshapeInp))\n",
    "        if reshapeTar is not None: inputSet = targetSet.reshape((size, *reshapeTar))\n",
    "        self.inputs, self.targets = NumpyDataCache(inputSet), NumpyDataCache(targetSet)\n",
    "        self.size: int = size\n",
    "        self.inpShape, self.tarShape = inputSet.shape[1:], targetSet.shape[1:]\n",
    "        self.pointer: int = 0\n",
    "        self.block: bool = False\n",
    "        self.batchSize: int = 1\n",
    "        self.indices = list(range(self.size))\n",
    "    @staticmethod\n",
    "    def oneHotEncode(_1dArray, oneHotMax=None):\n",
    "        if oneHotMax is None: oneHotMax = max(_1dArray) + 1\n",
    "        hotEncodedArray = np.zeros((len(_1dArray), oneHotMax, 1))\n",
    "        hotEncodedArray[np.arange(hotEncodedArray.shape[0]), _1dArray] = 1\n",
    "        return hotEncodedArray, oneHotMax\n",
    "    @staticmethod\n",
    "    def oneHotDecode(_3dArray):\n",
    "        return np.where(_3dArray == 1)[1]\n",
    "\n",
    "    # normalize input and target sets within the range of -scale to +scale\n",
    "    @staticmethod\n",
    "    def normalize(data, scale: float = None) -> tuple[\"np.ndarray\", float]:\n",
    "        if scale is None:\n",
    "            factor = 1\n",
    "        else:\n",
    "            factor = ne.evaluate(\"abs(data) * scale\", local_dict={'data': data, 'scale': scale}).max()\n",
    "        return data / factor, factor\n",
    "\n",
    "    # shuffle the index order\n",
    "    def randomize(self) -> \"None\":\n",
    "        np.random.shuffle(self.indices)\n",
    "\n",
    "    # returns a generator for input and target sets, each batch-sets of size batchSize at a time\n",
    "    # send signal '-1' to end generator\n",
    "    def batchGenerator(self, batchSize) -> Generator[tuple[\"np.ndarray\", \"np.ndarray\"], None, None]:\n",
    "        if self.block:\n",
    "            raise PermissionError(\"Access Denied: DataBase currently in use, \"\n",
    "                                  \"end previous generator before creating a new one\\n\"\n",
    "                                  \"send signal '-1' to end generator or reach StopIteration\")\n",
    "        self.block = True\n",
    "        self.batchSize = batchSize\n",
    "        self.randomize()\n",
    "        def generator() -> Generator:\n",
    "            signal = yield\n",
    "            while True:\n",
    "                if signal == -1 or self.pointer + batchSize >= self.size:\n",
    "                    rVal = self.__batch()\n",
    "                    self.__resetVars()\n",
    "                    yield rVal\n",
    "                    return\n",
    "                signal = yield self.__batch()\n",
    "                self.pointer += batchSize\n",
    "        gen = generator()\n",
    "        gen.send(None)\n",
    "        return gen\n",
    "    def __batch(self) -> tuple[np.ndarray, np.ndarray]:\n",
    "        indices = self.indices[self.pointer:self.pointer + self.batchSize]\n",
    "        inputBatch = self.inputs[indices]\n",
    "        targetBatch = self.targets[indices]\n",
    "        return inputBatch, targetBatch\n",
    "\n",
    "    # resets generator flags after generator cycle\n",
    "    def __resetVars(self):\n",
    "        self.pointer = 0\n",
    "        self.block = False\n",
    "        self.batchSize = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class PlotDataBase(Plot):\n",
    "    @staticmethod\n",
    "    def showMap():\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/Topologies/__init__.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from ..tools import Collections\n",
    "pass  # from .dataBase import DataBase, PlotDataBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Activators(Collections):\n",
    "    pass  # from .activationFunction import BaseActivationFunction, Sigmoid, TanH, Relu, PRelu, Elu, SoftMax, SoftPlus\n",
    "    Base, Sigmoid, TanH, Relu, PRelu, Elu, SoftMax, SoftPlus = \\\n",
    "        BaseActivationFunction, Sigmoid, TanH, Relu, PRelu, Elu, SoftMax, SoftPlus\n",
    "    def __init__(self, *activationFunctions: \"Activators.Base\"):\n",
    "        super(Activators, self).__init__(*activationFunctions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Initializers(Collections):\n",
    "    pass  # from .initializer import BaseInitializer, Uniform, Normal, Xavier, NormalizedXavier\n",
    "    Base, Uniform, Normal, Xavier, NormalizedXavier = \\\n",
    "        BaseInitializer, Uniform, Normal, Xavier, NormalizedXavier\n",
    "    def __init__(self, *initializer: \"Initializers.Base\"):\n",
    "        super(Initializers, self).__init__(*initializer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Optimizers(Collections):\n",
    "    pass  # from .optimizer import BaseOptimizer, GradientDecent, Decay, Momentum, NesterovMomentum, AdaGrad, RmsProp, AdaDelta, Adam\n",
    "    Base, GradientDecent, Decay, Momentum, NesterovMomentum, AdaGrad, RmpProp, AdaDelta, Adam = \\\n",
    "        BaseOptimizer, GradientDecent, Decay, Momentum, NesterovMomentum, AdaGrad, RmsProp, AdaDelta, Adam\n",
    "    def __init__(self, *optimizers: \"Optimizers.Base\"):\n",
    "        super(Optimizers, self).__init__(*optimizers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class LossFunction:\n",
    "    pass  # from .lossFunction import BaseLossFunction, MeanSquare\n",
    "    Base, MeanSquare = BaseLossFunction, MeanSquare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "__all__ = [\n",
    "    \"Activators\", \"Initializers\", \"LossFunction\", \"Optimizers\",\n",
    "    \"DataBase\", \"PlotDataBase\"\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/NeuralNetworks/base.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import time\n",
    "pass  # import warnings\n",
    "pass  # import cProfile\n",
    "pass  # import traceback\n",
    "pass  # import pstats\n",
    "pass  # import os\n",
    "pass  # from abc import ABCMeta, abstractmethod\n",
    "pass  # from typing import TYPE_CHECKING, Union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if TYPE_CHECKING:\n",
    "    pass  # from ..tools import *\n",
    "    pass  # from ..Topologies import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from ..tools import MagicBase, MagicProperty, makeMetaMagicProperty, PrintCols, iterable, secToHMS, statPrinter\n",
    "pass  # from ..Topologies import Activators, Initializers, Optimizers, LossFunction, DataBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseShape(MagicBase, metaclass=makeMetaMagicProperty(ABCMeta)):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    def __repr__(self):\n",
    "        return f\"<{self.__class__.__name__}:{self.NUM_LAYERS}:{self.RAW_SHAPES}>\"\n",
    "    def __save__(self):\n",
    "        pass\n",
    "    def __getitem__(self, item):\n",
    "        shapes = self.RAW_SHAPES[item]\n",
    "        return self.__class__(*shapes) if isinstance(item, slice) and shapes else self.SHAPES[item]\n",
    "    def __hash__(self):\n",
    "        return hash(self.SHAPES)\n",
    "    def __init__(self, *shapes):\n",
    "        \"\"\"do not change the signature of __init__\"\"\"\n",
    "        self.RAW_SHAPES = shapes\n",
    "        self.SHAPES = self._formatShapes(shapes)\n",
    "        assert hash(self.SHAPES)\n",
    "        self.NUM_LAYERS = len(self.SHAPES)\n",
    "        self.INPUT = self.SHAPES[0]\n",
    "        self.HIDDEN = self.SHAPES[1:-1]\n",
    "        self.OUTPUT = self.SHAPES[-1]\n",
    "    @staticmethod\n",
    "    @abstractmethod\n",
    "    def _formatShapes(shapes) -> tuple:\n",
    "        \"\"\"\n",
    "        method to format given shapes\n",
    "        :return: hashable formatted shapes\n",
    "        \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class UniversalShape(BaseShape):\n",
    "    \"\"\"Allows any shape format, creates 'BaseShape' like object\"\"\"\n",
    "    @staticmethod\n",
    "    def _formatShapes(shapes) -> tuple:\n",
    "        if iterable(shapes):\n",
    "            assert len(shapes) > 0\n",
    "            formattedShape = []\n",
    "            for s in shapes:\n",
    "                formattedShape.append(UniversalShape._formatShapes(s))\n",
    "            return tuple(formattedShape)\n",
    "        else:\n",
    "            return shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseLayer(MagicBase, metaclass=makeMetaMagicProperty(ABCMeta)):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    def __repr__(self):\n",
    "        return f\"<{self.__class__.__name__}:{self.SHAPE}: Ini={self.INITIALIZER}: Opt={self.optimizer}: \" \\\n",
    "               f\"AF={self.ACTIVATION_FUNCTION}>\"\n",
    "    def __str__(self):\n",
    "        DEPS = ': '.join(f\"{dName}:shape{getattr(self, dName).shape}\" for dName in self.DEPS)\n",
    "        return f\"{self.__repr__()[:-1]}:\\n{DEPS=}>\"\n",
    "    def __save__(self):\n",
    "        pass\n",
    "    def __init__(self, shape: \"BaseShape\",\n",
    "                 initializer: \"Initializers.Base\",\n",
    "                 optimizer: \"Optimizers.Base\",\n",
    "                 activationFunction: \"Activators.Base\",\n",
    "                 *depArgs, **depKwargs):\n",
    "        \"\"\"\n",
    "        :param shape: input, output, intermediate(optional) structure of the layer\n",
    "        \"\"\"\n",
    "        self.SHAPE = shape\n",
    "        self.INITIALIZER = initializer\n",
    "        self.optimizer = optimizer\n",
    "        self.ACTIVATION_FUNCTION = activationFunction\n",
    "        self.input = np.zeros((1, *self.SHAPE[0]), dtype=np.float32)\n",
    "        self.output = np.zeros((1, *self.SHAPE[-1]), dtype=np.float32)\n",
    "        self.inputDelta = np.zeros((1, *self.SHAPE[-1]), dtype=np.float32)\n",
    "        self.outputDelta = np.zeros((1, *self.SHAPE[0]), dtype=np.float32)\n",
    "        self.DEPS = self._defineDeps(*depArgs, **depKwargs)\n",
    "    def forPass(self, _input: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        f\"\"\"\n",
    "        method for forward pass of inputs\n",
    "        :param _input: self.output from the lower layer\n",
    "        :return: {self.output}\n",
    "        \"\"\"\n",
    "        self.input = _input\n",
    "        self.output = self._fire()\n",
    "        return self.output\n",
    "    def backProp(self, _delta: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        f\"\"\"\n",
    "        method for back propagation of deltas\n",
    "        :param _delta: value for {self.inputDelta} from {self.outputDelta} of the higher layer\n",
    "        :return: {self.outputDelta}\n",
    "        \"\"\"\n",
    "        self.inputDelta = _delta\n",
    "        self.outputDelta = self._wire()\n",
    "        return self.outputDelta\n",
    "    def changeOptimizer(self, optimizer: \"Optimizers.Base\"):\n",
    "        self.optimizer = optimizer\n",
    "        self._initializeDepOptimizer()\n",
    "    @abstractmethod\n",
    "    def _initializeDepOptimizer(self):\n",
    "        f\"\"\"create new optimizer instance for each dep in {self.DEPS} by using {self.optimizer.__new_copy__()}\"\"\"\n",
    "    @abstractmethod\n",
    "    def _defineDeps(self, *depArgs, **depKwargs) -> list['str']:\n",
    "        f\"\"\"\n",
    "        define all dependant objects ($DEPS) for the layer\n",
    "        :return: value for {self.DEPS}\n",
    "        \"\"\"\n",
    "    @abstractmethod\n",
    "    def _fire(self) -> \"np.ndarray\":\n",
    "        f\"\"\"\n",
    "        :return: value for {self.output}, is input for the higher layer\n",
    "        \"\"\"\n",
    "    @abstractmethod\n",
    "    def _wire(self) -> \"np.ndarray\":\n",
    "        f\"\"\"\n",
    "        :return: value for {self.outputDelta}, is delta for the lower layer\n",
    "        \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BasePlot(MagicBase, metaclass=makeMetaMagicProperty(ABCMeta)):\n",
    "    \"\"\"\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Network:\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    def __repr__(self):\n",
    "        LossFunction = self.LOSS_FUNCTION  # noqa\n",
    "        return f\"{self.__class__.__name__}:{LossFunction=}\"\n",
    "    def __str__(self):\n",
    "        layers = \"\\n\\t\\t\".join(repr(layer) for layer in self.LAYERS)\n",
    "        return f\"{super(Network, self).__str__()}:\\n\\t\\t{layers}\"\n",
    "    def __save__(self):\n",
    "        pass\n",
    "    def __init__(self, inputLayer: \"BaseLayer\", *layers: \"BaseLayer\", lossFunction: \"LossFunction.Base\"):\n",
    "        assert len(layers) > 0\n",
    "        self.LAYERS = inputLayer, *layers\n",
    "        self.INPUT_LAYER = inputLayer\n",
    "        self.HIDDEN_LAYERS = layers[:-1]\n",
    "        self.OUTPUT_LAYER = layers[-1]\n",
    "        self.LOSS_FUNCTION = lossFunction\n",
    "    def changeOptimizer(self, _optimizer: Union[\"Optimizers.Base\", \"Optimizers\"], index: int = None):\n",
    "        f\"\"\"\n",
    "        changes optimizer at index if given else changes all the optimizers to {_optimizer} or \n",
    "        uses given collection {Optimizers}\n",
    "        \"\"\"\n",
    "        assert isinstance(_optimizer, (Optimizers, Optimizers.Base))\n",
    "        if index is None:\n",
    "            optimizers = _optimizer.get(len(self.LAYERS)) if isinstance(_optimizer, Optimizers) else \\\n",
    "                (_optimizer,) * len(self.LAYERS)\n",
    "            for i, layer in enumerate(self.LAYERS):\n",
    "                layer.changeOptimizer(optimizers[i])\n",
    "        else:\n",
    "            layer: \"BaseLayer\" = self.LAYERS[index]\n",
    "            layer.changeOptimizer(_optimizer)\n",
    "    def forwardPass(self, _input) -> \"np.ndarray\":\n",
    "        f\"\"\"\n",
    "        calls(and sends hierarchical I/O) the forPass method of all the layers\n",
    "        :param _input: input for {self.INPUT_LAYER}\n",
    "        :return: output of {self.OUTPUT_LAYER}\n",
    "        \"\"\"\n",
    "        _output = self.INPUT_LAYER.forPass(_input)\n",
    "        for layer in self.HIDDEN_LAYERS: _output = layer.forPass(_output)\n",
    "        return self.OUTPUT_LAYER.forPass(_output)\n",
    "    def backPropagation(self, _delta) -> \"np.ndarray\":\n",
    "        f\"\"\"\n",
    "        calls(and sends hierarchical I/O) the backProp method of all the layers\n",
    "        :param _delta: delta for {self.OUTPUT_LAYER}\n",
    "        :return: delta of {self.INPUT_LAYER}\n",
    "        \"\"\"\n",
    "        _delta = self.OUTPUT_LAYER.backProp(_delta)\n",
    "        for reversedLayer in self.HIDDEN_LAYERS[::-1]: _delta = reversedLayer.backProp(_delta)\n",
    "        return self.INPUT_LAYER.backProp(_delta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class BaseNN(MagicBase, metaclass=makeMetaMagicProperty(ABCMeta)):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    STAT_PRINT_INTERVAL = 1\n",
    "    __optimizers = Optimizers(Optimizers.Adam(), ..., Optimizers.AdaGrad())\n",
    "    @MagicProperty\n",
    "    def optimizers(self):\n",
    "        return self.__optimizers\n",
    "    @optimizers.setter\n",
    "    def optimizers(self, _optimizers: \"Optimizers\"):\n",
    "        self.__optimizers = _optimizers\n",
    "        self.NETWORK.changeOptimizer(self.__optimizers)\n",
    "    def __repr__(self):\n",
    "        Shape = self.SHAPE\n",
    "        Cost, Time, Epochs = self.costTrained, secToHMS(self.timeTrained), self.epochTrained\n",
    "        acc = int(self.testAccuracy), int(self.accuracyTrained)\n",
    "        return f\"<{self.__class__.__name__}:Acc={acc[0]}%,{acc[1]}%: {Cost=:07.4f}: {Time=}: {Epochs=}: {Shape=}>\"\n",
    "    def __str__(self):\n",
    "        Optimizers = self.optimizers  # noqa\n",
    "        TrainDataBase, TestDataBase = self.trainDataBase, self.testDataBase\n",
    "        return f\"{self.__repr__()[1:-1]}:\\n\\t{Optimizers=}\\n\\t{TrainDataBase=}\\n\\t{TestDataBase=}\\n\\t{self.NETWORK}\"\n",
    "    def __save__(self):\n",
    "        pass\n",
    "    def __init__(self, shape: \"BaseShape\",\n",
    "                 initializers: \"Initializers\" = None,\n",
    "                 activators: \"Activators\" = None,\n",
    "                 lossFunction: \"LossFunction.Base\" = None):\n",
    "        if initializers is None: initializers = Initializers(Initializers.Xavier(2), ..., Initializers.Xavier())\n",
    "        if activators is None: activators = Activators(Activators.PRelu(), ..., Activators.SoftMax())\n",
    "        if lossFunction is None: lossFunction = LossFunction.MeanSquare()\n",
    "        self.SHAPE = shape\n",
    "        self.costHistory, self.accuracyHistory = [], []\n",
    "        self.accuracyTrained = self.testAccuracy = 0\n",
    "        self.costTrained = self.timeTrained = self.epochTrained = 0\n",
    "        self.numEpochs = self.batchSize = 1\n",
    "        self.epoch = self.batch = 0\n",
    "        self.numBatches = None\n",
    "        self.training = self.profiling = False\n",
    "        self.trainDataBase = self.testDataBase = None\n",
    "        self.NETWORK = self._constructNetwork(initializers, activators, lossFunction)\n",
    "    @abstractmethod\n",
    "    def _constructNetwork(self, initializers: \"Initializers\" = None,\n",
    "                          activators: \"Activators\" = None,\n",
    "                          lossFunction: \"LossFunction.Base\" = None) -> \"Network\":\n",
    "        pass\n",
    "    def process(self, _input) -> \"np.ndarray\":\n",
    "        if self.training:\n",
    "            warnings.showwarning(\"processing while training in progress may have unintended conflicts\",\n",
    "                                 ResourceWarning, 'neuralNetwork.py->AbstractNeuralNetwork.process', 0)\n",
    "            return np.NAN\n",
    "        return self.NETWORK.forwardPass(np.array(_input))\n",
    "    def profile(self):\n",
    "        self.profiling = True\n",
    "        prof = cProfile.Profile()\n",
    "        prof.runctx(\"self._train()\", locals=locals(), globals=globals())\n",
    "        prof.print_stats('cumtime')\n",
    "        prof.dump_stats('profile.pstat')\n",
    "        with open('profile.txt', 'w') as stream:\n",
    "            stats = pstats.Stats('profile.pstat', stream=stream)\n",
    "            stats.sort_stats('cumtime')\n",
    "            stats.print_stats()\n",
    "        os.remove('profile.txt')\n",
    "        self.profiling = False\n",
    "    def _train(self):\n",
    "        statPrinter('Epoch', f\"0/{self.numEpochs}\", prefix=PrintCols.CBOLDITALICURL + PrintCols.CBLUE)\n",
    "        self.training = True\n",
    "        if self.epochTrained == 0:\n",
    "            loss, _, acc = self._trainer(self.trainDataBase[:self.batchSize])\n",
    "            trainCosts, trainAccuracies = [loss], [acc]\n",
    "        else:\n",
    "            trainCosts, trainAccuracies = [self.costHistory[-1][-1]], [self.accuracyHistory[-1][-1]]\n",
    "        for self.epoch in range(1, self.numEpochs + 1):\n",
    "            epochTime = nextPrintTime = 0\n",
    "            costTrained = accuracyTrained = 0\n",
    "            try:\n",
    "                for self.batch, _batch in enumerate(self.trainDataBase.batchGenerator(self.batchSize)):\n",
    "                    timeStart = time.time()\n",
    "                    loss, delta, acc = self._trainer(_batch)\n",
    "                    self.NETWORK.backPropagation(delta)\n",
    "                    costTrained += loss\n",
    "                    accuracyTrained += acc\n",
    "                    batchTime = time.time() - timeStart\n",
    "                    epochTime += batchTime\n",
    "                    if epochTime >= nextPrintTime or self.batch == self.numBatches - 1:\n",
    "                        nextPrintTime += self.STAT_PRINT_INTERVAL\n",
    "                        self.printStats(costTrained / (self.batch + 1), trainCosts[-1],\n",
    "                                        accuracyTrained / (self.batch + 1), trainAccuracies[-1], epochTime)\n",
    "                self.timeTrained += epochTime\n",
    "                self.epochTrained += 1\n",
    "                self.costTrained = costTrained / self.numBatches\n",
    "                self.accuracyTrained = accuracyTrained / self.numBatches\n",
    "                trainCosts.append(self.costTrained)\n",
    "                trainAccuracies.append(self.accuracyTrained)\n",
    "            except Exception:  # noqa\n",
    "                traceback.print_exc()\n",
    "                warnings.showwarning(\"unhandled exception occurred while training,\"\n",
    "                                     \"\\nquiting training and rolling back to previous auto save\", RuntimeWarning,\n",
    "                                     'base.py', 0)\n",
    "                raise NotImplementedError  # todo: roll back and auto save\n",
    "        self.costHistory.append(trainCosts)\n",
    "        self.accuracyHistory.append(trainAccuracies)\n",
    "        self.training = False\n",
    "        statPrinter('', '', end='\\n')\n",
    "    def train(self, epochs: int = None,\n",
    "              batchSize: int = None,\n",
    "              trainDataBase: \"DataBase\" = None,\n",
    "              optimizers: \"Optimizers\" = None,\n",
    "              profile: bool = False,\n",
    "              test: Union[bool, \"DataBase\"] = None):\n",
    "        # todo: implement \"runs\"\n",
    "        if epochs is not None: self.numEpochs = epochs\n",
    "        if batchSize is not None: self.batchSize = batchSize\n",
    "        if trainDataBase is not None: self.trainDataBase = trainDataBase\n",
    "        if optimizers is not None: self.optimizers = optimizers\n",
    "        assert isinstance(self.trainDataBase, DataBase)\n",
    "        assert self.trainDataBase.inpShape == self.SHAPE.INPUT and self.trainDataBase.tarShape == self.SHAPE.OUTPUT\n",
    "        if trainDataBase is not None or batchSize is not None:\n",
    "            self.numBatches = int(np.ceil(self.trainDataBase.size / self.batchSize))\n",
    "        if profile:\n",
    "            self.profile()\n",
    "        else:\n",
    "            self._train()\n",
    "        if test or test is None: self.test(test)\n",
    "    def printStats(self, loss, prevLoss, acc, prevAcc, epochTime):\n",
    "        print(end='\\r')\n",
    "        \"\"\"__________________________________________________________________________________________________________\"\"\"\n",
    "        statPrinter('Epoch', f\"{self.epoch:0{len(str(self.numEpochs))}d}/{self.numEpochs}\",\n",
    "                    prefix=PrintCols.CBOLDITALICURL + PrintCols.CBLUE, suffix='')\n",
    "        statPrinter('Batch', f\"{(b := self.batch + 1):0{len(str(self.numBatches))}d}/{self.numBatches}\",\n",
    "                    suffix='', end='')\n",
    "        statPrinter(f\"({int(b / self.numBatches * 100):03d}%)\", '')\n",
    "        \"\"\"__________________________________________________________________________________________________________\"\"\"\n",
    "        statPrinter('Cost', f\"{loss:07.4f}\", prefix=PrintCols.CYELLOW, suffix='')\n",
    "        statPrinter('Cost-Dec', f\"{(prevLoss - loss):07.4f}\", suffix='')\n",
    "        statPrinter('Acc', f\"{int(acc):03d}%\", prefix=PrintCols.CYELLOW, suffix='')\n",
    "        statPrinter('Acc-Inc', f\"{int(acc - prevAcc):03d}%\")\n",
    "        \"\"\"__________________________________________________________________________________________________________\"\"\"\n",
    "        elapsed = self.timeTrained + epochTime\n",
    "        avgTime = elapsed / (effectiveEpoch := self.epoch - 1 + (self.batch + 1) / self.numBatches)\n",
    "        statPrinter('Time', secToHMS(elapsed), prefix=PrintCols.CBOLD + PrintCols.CRED2, suffix='')\n",
    "        statPrinter('Epoch-Time', secToHMS(epochTime), suffix='')\n",
    "        statPrinter('Avg-Time', secToHMS(avgTime), suffix='')\n",
    "        statPrinter('Eta', secToHMS(avgTime * (self.numEpochs - effectiveEpoch)))\n",
    "        \"\"\"__________________________________________________________________________________________________________\"\"\"\n",
    "    def _trainer(self, _batch: tuple[\"np.ndarray\", \"np.ndarray\"]):\n",
    "        output, target = self.NETWORK.forwardPass(_batch[0]), _batch[1]\n",
    "        loss, delta = self.NETWORK.LOSS_FUNCTION(output, target)\n",
    "        acc = self._tester(output, target)\n",
    "        return loss, delta, acc\n",
    "    def accuracy(self, inputSet, targetSet):\n",
    "        assert (size := np.shape(inputSet)[0]) == np.shape(targetSet)[0], \\\n",
    "            \"the size of both inputSet and targetSet should be same\"\n",
    "        try:\n",
    "            return self._tester(self.process(inputSet), targetSet)\n",
    "        except MemoryError:\n",
    "            accuracy1 = self.accuracy(inputSet[:(to := size // 2)], targetSet[:to])\n",
    "            accuracy2 = self.accuracy(inputSet[to:], targetSet[to:])\n",
    "            return (accuracy1 + accuracy2) / 2\n",
    "    def test(self, testDataBase: \"DataBase\" = None):\n",
    "        statPrinter('Testing', 'wait...', prefix=PrintCols.CBOLD + PrintCols.CYELLOW, suffix='')\n",
    "        if self.trainDataBase is not None:\n",
    "            self.accuracyTrained = self.accuracy(self.trainDataBase.inputs, self.trainDataBase.targets)\n",
    "        if testDataBase is not None:\n",
    "            assert isinstance(testDataBase, DataBase)\n",
    "            assert testDataBase.inpShape == self.SHAPE.INPUT and testDataBase.tarShape == self.SHAPE.OUTPUT\n",
    "            self.testAccuracy = self.accuracy(testDataBase.inputs, testDataBase.targets)\n",
    "        print(end='\\r')\n",
    "        statPrinter('Train-Accuracy', f\"{self.accuracyTrained}%\", suffix='', end='\\n')\n",
    "        statPrinter('Test-Accuracy', f\"{self.testAccuracy}%\", end='\\n')\n",
    "    @staticmethod\n",
    "    def _tester(_output: \"np.ndarray\", _target: \"np.ndarray\") -> \"np.ndarray\":\n",
    "        if np.shape(_target) != 1:\n",
    "            # poly node multi classification\n",
    "            outIndex = np.argmax(_output, axis=1)\n",
    "            targetIndex = np.argmax(_target, axis=1)\n",
    "        else:\n",
    "            # single node binary classification\n",
    "            outIndex = _output.round()\n",
    "            targetIndex = _target\n",
    "        result: \"np.ndarray\" = outIndex == targetIndex\n",
    "        return result.mean() * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/NeuralNetworks/dense.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from typing import TYPE_CHECKING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if TYPE_CHECKING:\n",
    "    pass  # from ..Topologies import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from .base import BaseShape, BaseLayer, BasePlot, BaseNN, UniversalShape, Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DenseShape(BaseShape):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    def __init__(self, *shapes: int):\n",
    "        super(DenseShape, self).__init__(*shapes)\n",
    "    @staticmethod\n",
    "    def _formatShapes(shapes) -> tuple:\n",
    "        assert len(shapes) > 0\n",
    "        formattedShape = []\n",
    "        for s in shapes:\n",
    "            assert isinstance(s, int) and s > 0, \"all args of *shapes must be integers > 0\"\n",
    "            formattedShape.append((s, 1))\n",
    "        return tuple(formattedShape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DenseLayer(BaseLayer):  # todo: pre-set deltas after forwardPass\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    def __save__(self):\n",
    "        return super(DenseLayer, self).__save__()\n",
    "    def _initializeDepOptimizer(self):\n",
    "        self.weightOptimizer = self.optimizer.__new_copy__()\n",
    "        self.biasesOptimizer = self.optimizer.__new_copy__()\n",
    "    def _defineDeps(self) -> list['str']:\n",
    "        self.weights = self.INITIALIZER(UniversalShape(self.SHAPE.INPUT, *(self.SHAPE.OUTPUT[0], self.SHAPE.INPUT[0]),\n",
    "                                                       self.SHAPE.OUTPUT))\n",
    "        self.biases = self.INITIALIZER(UniversalShape(self.SHAPE.INPUT, *(self.SHAPE.OUTPUT[0], 1), self.SHAPE.OUTPUT))\n",
    "        self.delta = None\n",
    "        self.activeDerivedDelta = None\n",
    "        self._initializeDepOptimizer()\n",
    "        return ['weights', 'biases']\n",
    "    def __gradWeights(self, weights):  # BottleNeck\n",
    "        self.delta = np.einsum('oi,...oj->...ij', weights, self.inputDelta, optimize='greedy')\n",
    "        self.activeDerivedDelta = \\\n",
    "            np.einsum('...ij,...ij->...ij', self.inputDelta, self.ACTIVATION_FUNCTION.activatedDerivative(self.output),\n",
    "                      optimize='greedy')\n",
    "        return np.einsum('...ij,...oj->oi', self.input, self.activeDerivedDelta, optimize='greedy')\n",
    "    def __gradBiases(self, _=None):\n",
    "        return self.activeDerivedDelta.sum(axis=0)\n",
    "    def _fire(self) -> \"np.ndarray\":  # BottleNeck\n",
    "        return self.ACTIVATION_FUNCTION.activation(\n",
    "            np.einsum('oi,...ij->...oj', self.weights, self.input, optimize='greedy') + self.biases)\n",
    "    def _wire(self) -> \"np.ndarray\":\n",
    "        self.weights -= self.weightOptimizer(self.__gradWeights, self.weights)\n",
    "        self.biases -= self.biasesOptimizer(self.__gradBiases, self.biases)\n",
    "        return self.delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DensePlot(BasePlot):\n",
    "    \"\"\"\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DenseNN(BaseNN):\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    def __str__(self):\n",
    "        return super(DenseNN, self).__str__()\n",
    "    def __save__(self):\n",
    "        return super(DenseNN, self).__save__()\n",
    "    def __init__(self, shape: \"DenseShape\",\n",
    "                 initializers: \"Initializers\" = None,\n",
    "                 activators: \"Activators\" = None,\n",
    "                 lossFunction: \"LossFunction.Base\" = None):\n",
    "        super(DenseNN, self).__init__(shape, initializers, activators, lossFunction)\n",
    "    def _constructNetwork(self, initializers: \"Initializers\" = None,\n",
    "                          activators: \"Activators\" = None,\n",
    "                          lossFunction: \"LossFunction.Base\" = None) -> \"Network\":\n",
    "        layers = []\n",
    "        for i, _initializer, _optimizer, _aF in zip(range(_length := self.SHAPE.NUM_LAYERS - 1),\n",
    "                                                    initializers(_length),\n",
    "                                                    self.optimizers(_length),  # noqa\n",
    "                                                    activators(_length)):\n",
    "            layers.append(DenseLayer(self.SHAPE[i:i + 2], _initializer, _optimizer, _aF))\n",
    "        return Network(*layers, lossFunction=lossFunction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/NeuralNetworks/conv.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from typing import TYPE_CHECKING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "if TYPE_CHECKING:\n",
    "    pass  # from ..tools import *\n",
    "    pass  # from ..Topologies import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from .base import BaseShape, BaseLayer, BasePlot, BaseNN, UniversalShape, Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class ConvShape:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class ConvLayer:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class ConvPlot(BasePlot):\n",
    "    \"\"\"\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class ConvNN:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/NeuralNetworks/__init__.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Base:\n",
    "    pass  # from .base import BaseShape, BaseShape, BasePlot, BaseNN, UniversalShape, Network\n",
    "    Shape, Layer, Plot, NN = BaseShape, BaseShape, BasePlot, BaseNN\n",
    "    UniversalShape, Network = UniversalShape, Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Dense:\n",
    "    pass  # from .dense import DenseShape, DenseLayer, DensePlot, DenseNN\n",
    "    Shape, Layer, Plot, NN = DenseShape, DenseLayer, DensePlot, DenseNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Conv:\n",
    "    pass  # from .conv import ConvShape, ConvLayer, ConvPlot, ConvNN\n",
    "    Shape, Layer, Plot, NN = ConvShape, ConvLayer, ConvPlot, ConvNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "__all__ = [\n",
    "    \"Base\", \"Dense\", \"Conv\"\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /__init__/__init__.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from .NeuralNetworks import *\n",
    "pass  # from .tools import *\n",
    "pass  # from .Topologies import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# /main.py<br>\n",
    "todo: dynamic optimizers<br>\n",
    "todo: DataBase shaping using self.SHAPE<br>\n",
    "todo: auto hyperparameter tuning: Grid search, Population-based natural selection<br>\n",
    "todo: auto train stop, inf train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "pass  # from __init__ import *\n",
    "pass  # from DataSets import dataSet\n",
    "pass  # from Models import model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "db = DataBase.load(dataSet.TrainSets.EmnistBalanced, normalizeInp=1, reshapeInp=(-1, 1),\n",
    "                   name='TrainSets.EmnistBalanced')\n",
    "db2 = DataBase.load(dataSet.TestSets.EmnistBalanced, normalizeInp=1, reshapeInp=(-1, 1),\n",
    "                    name='TestSets.EmnistBalanced')\n",
    "# db2 = False\n",
    "dense_nn = Dense.NN(shape=Dense.Shape(db.inpShape[0], *(392, 196), db.tarShape[0]),\n",
    "                    initializers=None,\n",
    "                    activators=None,\n",
    "                    lossFunction=None)\n",
    "dense_nn.train(epochs=10,\n",
    "               batchSize=256,\n",
    "               trainDataBase=db,\n",
    "               optimizers=None,\n",
    "               profile=False,\n",
    "               test=db2)\n",
    "print(db, db2, dense_nn, sep='\\n\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}