Forex Regularized Models Version 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5c0d0bd1-3126-435e-b70b-4896395b0b39",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: catboost in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (1.2.5)\n",
      "Requirement already satisfied: graphviz in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (0.20.1)\n",
      "Requirement already satisfied: matplotlib in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (3.5.3)\n",
      "Requirement already satisfied: numpy>=1.16.0 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (1.21.6)\n",
      "Requirement already satisfied: pandas>=0.24 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (1.3.5)\n",
      "Requirement already satisfied: scipy in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (1.7.3)\n",
      "Requirement already satisfied: plotly in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (5.14.1)\n",
      "Requirement already satisfied: six in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from catboost) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from pandas>=0.24->catboost) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from pandas>=0.24->catboost) (2023.3)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from matplotlib->catboost) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from matplotlib->catboost) (4.38.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from matplotlib->catboost) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from matplotlib->catboost) (23.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from matplotlib->catboost) (8.1.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from matplotlib->catboost) (3.0.9)\n",
      "Requirement already satisfied: tenacity>=6.2.0 in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from plotly->catboost) (8.2.2)\n",
      "Requirement already satisfied: typing-extensions in /home/jupyterlab/conda/envs/python/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib->catboost) (4.5.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install catboost\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "813a5b69-c85d-4955-ac54-ce25b4892c5b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "6365ebf6-f6c0-48d1-8915-d8d1ef498303",
   "metadata": {},
   "source": [
    "# Train Forex Models With Regularized Parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3767ae7c-5f70-422a-9e0d-e902f423c8c4",
   "metadata": {},
   "source": [
    "## Version 2\n",
    "* Temporal features False (Month, Day of week, Week Of Month)\n",
    "* Hour 4 Features Lag by 7 optional\n",
    "* Day Features (RSI, RSI crossover, RSI-Based SMA, ) Lag by 9\n",
    "* Starting Date June 2020 Training Till September 2023\n",
    "* Ending Date May 2024\n",
    "* Hour 4 Features 'relative range', 'candle type', 'heikin ashi' \n",
    "* Hyperparameters Regularized\n",
    "* Removing CHF\n",
    "\n",
    "* Evaluation Metric \n",
    "* Sharp Ratio \n",
    "* Accuracy by net gains\n",
    "\n",
    "Version 1 Regularized On Two Months April May On Two Conditions :\n",
    " \n",
    " 1- hyper parameters chosen should give positive gains for any currency pairs for those two months\n",
    " \n",
    " 2- Previous months accuracy should be 50% and net gains accuracy close to 50% or 52%\n",
    "\n",
    "Results :- (pips)\n",
    "* ❌ December -3080\n",
    "* ✅ January 4844\n",
    "* ✅ February 1288\n",
    "* ✅ March 1883\n",
    "* ✅ April \n",
    "* ✅ May\n",
    "\n",
    "\n",
    "📉 Not all models were in positive gains in previous month despite all performing good on april may with positive gains\n",
    "\n",
    "⭐ But parameters performing good on two different months gave them ability to keep majority of currency pairs model in positive gain\n",
    "\n",
    "💢 Successfully managed to make December as profitable even though not recommended for trading by trading community.\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a10584b4-5c8e-420d-86f5-07b937a839da",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "results_dict = {}\n",
    "day_features, hour4_features = load_features_files()\n",
    "forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY'\n",
    "        ]\n",
    "\n",
    "symbol_hyperparameter = [\n",
    " 'AUDJPY', 'AUDUSD', 'AUDCAD', 'AUDCHF', 'AUDNZD',\n",
    " 'CADJPY', \n",
    " 'EURAUD', 'EURCAD', 'EURUSD', 'EURGBP', 'EURNZD',\n",
    " 'GBPCAD', 'GBPCHF', 'GBPUSD', 'GBPNZD',\n",
    " 'NZDCHF', 'NZDJPY',\n",
    " 'USDCHF', 'USDJPY', 'USDCAD',\n",
    " 'CADCHF', \n",
    " 'NZDCAD', 'NZDUSD']\n",
    "\n",
    "# symbols = ['USDCAD']\n",
    "\n",
    "for symbol in forex_pairs:\n",
    "    \n",
    "    \n",
    "    day_data, hour4_data = get_day_hour4_features(symbol, day_features, hour4_features)\n",
    "\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test = get_features_transformed(symbol, day_data, hour4_data)\n",
    "    \n",
    "    print(symbol)\n",
    "    # model = load_model(symbol)\n",
    "    \n",
    "    if symbol in symbol_hyperparameter: \n",
    "      \n",
    "       symbol_parameters = load_parameters(symbol)\n",
    "       iteration, learning_rate, depth = symbol_parameters['Iterations'], symbol_parameters['Lr'], symbol_parameters['depth']\n",
    "       parameters = [ iteration, learning_rate, depth ] \n",
    "       model = train_model(X_train_scaled, X_test_scaled, y_train, y_test, parameters=parameters)\n",
    "    \n",
    "    else:\n",
    "      model = train_model(X_train_scaled, X_test_scaled, y_train, y_test)\n",
    "\n",
    "    # model = load_model(symbol)\n",
    "    gains , accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled,  y_test)\n",
    "    \n",
    "     # 2  # 1 --> May 2024 , # 2 --> April 2024\n",
    "    step = 1 # ____          May \n",
    "    # step = 2 # ____       April\n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster] ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "    \n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    results_dict[symbol] = { 'net_gains': int(net_gains), 'accuracy': int(accuracy), 'month': 'may', \n",
    "                             'accuracy_by_net_gains': int(accuracy_by_net_gains)  }\n",
    "    save_model(symbol, model)\n",
    "    \n",
    "\n",
    "    # accuracy_by_net_gains = gains['accuracy_by_net_gains_before']\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    # print('accuracy_by_net_gains \\n',accuracy_by_net_gains )\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5419704-7f8f-4900-b302-a2a0cfefa4f9",
   "metadata": {},
   "source": [
    "# Code Structure To See Model Accuracy For A Symbol "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "632d3051-347e-46ae-826f-0e993bd0bc81",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "day_features, hour4_features = load_features_files()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "571c259f-b6b2-4cd0-9f37-61e32a1cf4bd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/pandas/core/algorithms.py:1601: FutureWarning: Comparison of Timestamp with datetime.date is deprecated in order to match the standard library behavior.  In a future version these will be considered non-comparable.Use 'ts == pd.Timestamp(date)' or 'ts.date() == date' instead.\n",
      "  return arr.searchsorted(value, side=side, sorter=sorter)\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:35: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:35: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:42: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:42: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/pandas/core/generic.py:6392: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return self._update_inplace(result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GBPJPY\n",
      "0:\tlearn: 97.5604005\ttotal: 127ms\tremaining: 1.14s\n",
      "9:\tlearn: 86.7140820\ttotal: 1.23s\tremaining: 0us\n",
      "Mean Squared Error: 9102.590768458482\n",
      "saved successfully\n",
      "MAY\n",
      "GBPJPY  Starting date : 2024-05-01 \n",
      "['+1-1', '+1-1', '-1+1', 1, 1, 1, 1, 1, 1, '+1-1', '-1+1', '-1+1', '-1+1', '+1-1', 1, -1, 1, 1, 1, '+1-1', '+1-1', '-1+1']\n",
      "Sharpe Ratio: -0.08\n",
      "Pips On Profit Side was :  607.1000000000055\n",
      "Pips On Loss Side was :  726.1999999999972\n",
      "accuracy by days 50.0\n",
      "accuracy_by_net_gains  45.533638340958845\n",
      "net profit  -119.09999999999172\n",
      "\n",
      "accuracy by net gains on previous test data  56.778204609785696\n",
      "accuracy on all test points excluding last 22 points  55.61497326203209\n",
      "APRIL\n",
      "GBPJPY  Starting date : 2024-04-01 \n",
      "['-1+1', '-1+1', -1, 1, 1, '-1+1', '+1-1', 1, -1, '-1+1', 1, '-1+1', 1, '+1-1', 1, 1, 1, 1, 1, -1, 1, '+1-1']\n",
      "Sharpe Ratio: 0.14\n",
      "Pips On Profit Side was :  1219.800000000009\n",
      "Pips On Loss Side was :  820.7999999999998\n",
      "accuracy by days 63.63636363636363\n",
      "accuracy_by_net_gains  59.77653631284934\n",
      "net profit  399.0000000000092\n",
      "\n",
      "accuracy by net gains on previous test data  56.26005657085753\n",
      "accuracy on all test points excluding last 22 points  54.54545454545454\n",
      "MARCH\n",
      "GBPJPY  Starting date : 2024-02-29 \n",
      "['-1+1', 1, '+1-1', '+1-1', '+1-1', '+1-1', '+1-1', 1, 1, 1, '-1+1', '-1+1', 1, 1, -1, '+1-1', '-1+1', 1, '+1-1', '+1-1', '+1-1', '+1-1']\n",
      "Sharpe Ratio: 0.02\n",
      "Pips On Profit Side was :  711.6000000000014\n",
      "Pips On Loss Side was :  669.8000000000008\n",
      "accuracy by days 36.36363636363637\n",
      "accuracy_by_net_gains  51.51295786882875\n",
      "net profit  41.80000000000064\n",
      "\n",
      "accuracy by net gains on previous test data  56.88897840181071\n",
      "accuracy on all test points excluding last 22 points  57.34265734265735\n",
      "Iterations:  10\n",
      "Learning rate:  0.1\n",
      "Depth:  10\n",
      "FEB\n",
      "GBPJPY  Starting date : 2024-01-30 \n",
      "[-1, 1, 1, '+1-1', 1, 1, '-1+1', 1, 1, 1, '+1-1', '+1-1', 1, '+1-1', 1, 1, 1, 1, 1, '+1-1', '+1-1', '+1-1']\n",
      "Sharpe Ratio: 0.10\n",
      "Pips On Profit Side was :  666.8000000000035\n",
      "Pips On Loss Side was :  520.5000000000013\n",
      "accuracy by days 63.63636363636363\n",
      "accuracy_by_net_gains  56.16103764844612\n",
      "net profit  146.30000000000223\n",
      "\n",
      "accuracy by net gains on previous test data  56.98252069917202\n",
      "accuracy on all test points excluding last 22 points  56.19834710743802\n",
      "JAN\n",
      "GBPJPY  Starting date : 2023-12-29 \n",
      "[-1, 1, 1, '-1+1', '-1+1', '+1-1', 1, -1, '+1-1', '-1+1', 1, 1, 1, '+1-1', 1, 1, -1, '+1-1', 1, '+1-1', -1]\n",
      "Sharpe Ratio: 0.40\n",
      "Pips On Profit Side was :  1110.7999999999977\n",
      "Pips On Loss Side was :  317.999999999995\n",
      "accuracy by days 59.09090909090909\n",
      "accuracy_by_net_gains  77.7435610302354\n",
      "net profit  792.8000000000027\n",
      "\n",
      "accuracy by net gains on previous test data  53.18473376265889\n",
      "accuracy on all test points excluding last 22 points  55.55555555555556\n",
      "DEC\n",
      "GBPJPY  Starting date : 2023-11-29 \n",
      "[1, '+1-1', '+1-1', '+1-1', -1, -1, 1, 1, '+1-1', '+1-1', 1, '+1-1', 1, 1, '+1-1', '+1-1', 1, -1, 1, 1, '+1-1', -1]\n",
      "Sharpe Ratio: 0.05\n",
      "Pips On Profit Side was :  1154.9000000000033\n",
      "Pips On Loss Side was :  1023.5000000000014\n",
      "accuracy by days 59.09090909090909\n",
      "accuracy_by_net_gains  53.015975027543185\n",
      "net profit  131.4000000000019\n",
      "\n",
      "accuracy by net gains on previous test data  53.25000443868397\n",
      "accuracy on all test points excluding last 22 points  54.54545454545454\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def test_model(symbol, iteration, learning_rate, depth, window_size=9, alpha_type = \"gamma\", save_model=False ):\n",
    "\n",
    "    day_data, hour4_data = get_day_hour4_features(symbol, day_features, hour4_features)\n",
    "\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test, X_test = get_features_transformed(symbol, day_data, hour4_data,  X_test_re=True,  window_size=window_size, alpha_type=alpha_type ) \n",
    "\n",
    "    print(symbol)\n",
    "    # model = load_model(symbol)\n",
    "\n",
    "\n",
    "    # symbol_parameters = load_parameters(symbol)\n",
    "    # iteration, learning_rate, depth = symbol_parameters['Iterations'], symbol_parameters['Lr'], symbol_parameters['depth']\n",
    "\n",
    "\n",
    "    # iteration, learning_rate, depth = 15, 0.68, 6 # AUDNZD\n",
    "\n",
    "    # iteration, learning_rate, depth = 210, 0.19, 7 # AUDCAD\n",
    "\n",
    "    # iteration, learning_rate, depth = 17, 0.9, 7 # AUDCAD\n",
    "\n",
    "\n",
    "\n",
    "    parameters = [ iteration, learning_rate, depth ]\n",
    "    \n",
    "    # model = finetune_model(X_train_scaled, X_test_scaled, y_train, y_test, parameters=parameters)\n",
    "    model = train_model(X_train_scaled, X_test_scaled, y_train, y_test, parameters=parameters)\n",
    "    if save_model == True : save_model_v2(symbol, model, parameters, alpha_type, window_size)\n",
    "    # if save_model == True :     \n",
    "    #     save_model(symbol, model)\n",
    "\n",
    "        \n",
    "\n",
    "    print(\"MAY\")\n",
    "\n",
    "    step = 1 # ____          May \n",
    "    # step = 2 # ____       April\n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster], X_test=X_test ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "\n",
    "    print(\"APRIL\")\n",
    "    step = 2 # ____       April\n",
    "    cluster = 22\n",
    "    # cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster], X_test=X_test ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "\n",
    "\n",
    "\n",
    "    print(\"MARCH\")\n",
    "    step = 3 # ____      March\n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster], X_test=X_test ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "    \n",
    "    print(\"Iterations: \",iteration)\n",
    "    print(\"Learning rate: \",learning_rate) \n",
    "    print(\"Depth: \",depth)\n",
    "    \n",
    "    \n",
    "    print(\"FEB\")\n",
    "    step = 4 # ____      Feb\n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster], X_test=X_test ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "    \n",
    "    print(\"JAN\")\n",
    "    step = 5 # ____      Jan\n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster], X_test=X_test ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "\n",
    "    print(\"DEC\")\n",
    "    step = 6 # ____      Jan\n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster], X_test=X_test ) # custom_sample= None, last_cluster = None  \n",
    "    net_gains = gains['net_gains']\n",
    "\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n",
    "    \n",
    "    \n",
    "# symbol = \"AUDCAD\"\n",
    "\n",
    "# test_model(symbol=\"USDCAD\", iteration=101, learning_rate=0.59, depth=7 ) # Good For Multiple Months\n",
    "# test_model(symbol=\"EURUSD\", iteration=19, learning_rate=0.49, depth=7 ) # Good For Multiple Months Surplus For Jan, Feb, Mar, Apr\n",
    "# test_model(symbol=\"AUDJPY\", iteration=11, learning_rate=0.01, depth=5 ) # Good For Multiple Months Surplus For Jan, Feb, Mar, Apr\n",
    "# test_model(symbol=\"CHFJPY\", iteration=21, learning_rate=0.69, depth=8 ) # Good For Multiple Months Surplus For Jan, Feb, Mar, Apr\n",
    "# test_model(symbol=\"EURUSD\", iteration=39, learning_rate=0.01, depth=7 )\n",
    "\n",
    "\n",
    "# Reformed OneZ With Depth 10-11 And Moderate Learning Rate \n",
    "# Hour 4 Features Of Last 3 days Only\n",
    "\n",
    "# ====================\n",
    "# V2 models\n",
    "# works when period = 45 in harmonic mean low and harmonic mean high\n",
    "\n",
    "# test_model(symbol=\"USDJPY\", iteration=3, learning_rate=0.2, depth=12, window_size=7, save_model=True) # version A passed\n",
    "# test_model(symbol=\"USDJPY\", iteration=3, learning_rate=0.23, depth=12, window_size=7) # version A\n",
    "\n",
    "\n",
    "# test_model(symbol=\"EURAUD\", iteration=7, learning_rate=0.4, depth=7, window_size=8) # version passed\n",
    "# test_model(symbol=\"EURAUD\", iteration=33, learning_rate=0.15, depth=10, window_size=8, save_model=True) # version -> A passed\n",
    "\n",
    "# test_model(symbol=\"EURAUD\", iteration=3, learning_rate=0.15, depth=11, window_size=11) # version \n",
    "\n",
    "# xxtest_model(symbol=\"EURAUD\", iteration=27, learning_rate=0.19, depth=10, window_size=8) # another version -> A\n",
    "# test_model(symbol=\"EURAUD\", iteration=27, learning_rate=0.15, depth=10, window_size=8) # another version -> A\n",
    "# test_model(symbol=\"EURAUD\", iteration=77, learning_rate=0.5, depth=9, window_size=10) # version \n",
    "# test_model(symbol=\"EURAUD\", iteration=100, learning_rate=0.05, depth=6, window_size=9) # version \n",
    "\n",
    "\n",
    "\n",
    "# xxtest_model(symbol=\"EURNZD\", iteration=9, learning_rate=0.25, depth=10, window_size=9) # version -> A\n",
    "# test_model(symbol=\"EURNZD\", iteration=11, learning_rate=0.15, depth=9, window_size=9, save_model=True) # version -> A passed\n",
    "\n",
    "# test_model(symbol=\"EURNZD\", iteration=66, learning_rate=0.51, depth=10, window_size=9) # another version\n",
    "\n",
    "\n",
    "# Pair EURNZD__________________________\n",
    "# test_model(symbol=\"EURNZD\", iteration=5, learning_rate=0.15, depth=10, window_size=6)   # another version\n",
    "# test_model(symbol=\"EURNZD\", iteration=150, learning_rate=0.02, depth=6, window_size=7) # another version\n",
    "# test_model(symbol=\"EURNZD\", iteration=300, learning_rate=0.01, depth=6, window_size=7) # another version\n",
    "\n",
    "\n",
    "#  Pair EURCAD________________________-\n",
    "# test_model(symbol=\"EURCAD\", iteration=1, learning_rate=0.10, depth=10, window_size=9) # version\n",
    "# xxtest_model(symbol=\"EURCAD\", iteration=50, learning_rate=0.05, depth=6, window_size=6, alpha_type=\"gamma\" ) # version A gamma \n",
    "# test_model(symbol=\"EURCAD\", iteration=13, learning_rate=0.05, depth=9, window_size=7, alpha_type=\"alpha\" ) # version A alpha\n",
    "# xx test_model(symbol=\"EURCAD\", iteration=5, learning_rate=0.10, depth=10, window_size=9, alpha_type=\"gamma\") # another version A\n",
    "\n",
    "# test_model(symbol=\"EURCAD\", iteration=22, learning_rate=0.25, depth=8, window_size=9) # version seems passed\n",
    "# test_model(symbol=\"EURCAD\", iteration=21, learning_rate=0.25, depth=8, window_size=9) # version A passed\n",
    "\n",
    "# test_model(symbol=\"EURCAD\", iteration=27, learning_rate=0.17, depth=7, window_size=6, alpha_type=\"beta\" ) # version seems passed\n",
    "# test_model(symbol=\"EURCAD\", iteration=28, learning_rate=0.17, depth=7, window_size=6, alpha_type=\"beta\", save_model=True ) # version A passed\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"EURUSD\", iteration=210, learning_rate=0.05, depth=6, window_size=7, alpha_type=\"gamma\", save_model=True ) #  version A passed\n",
    "# test_model(symbol=\"EURUSD\", iteration=5, learning_rate=0.05, depth=6, window_size=7, alpha_type=\"gamma\" ) #  another version \n",
    "# test_model(symbol=\"EURUSD\", iteration=1, learning_rate=0.25, depth=10, window_size=9, alpha_type=\"alpha\"  ) # another version\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"USDCAD\", iteration=3, learning_rate=0.11, depth=9, window_size=9, alpha_type=\"omega\") # another version \n",
    "# test_model(symbol=\"USDCAD\", iteration=1, learning_rate=0.20, depth=10, window_size=9, alpha_type=\"omega\") # another version\n",
    "# xxtest_model(symbol=\"USDCAD\", iteration=5, learning_rate=0.5, depth=10, window_size=11, alpha_type=\"gamma\") # another version \n",
    "# xxtest_model(symbol=\"USDCAD\", iteration=5, learning_rate=0.15, depth=10, window_size=11, alpha_type=\"gamma\") # version A 60 accuracy\n",
    "\n",
    "# test_model(symbol=\"USDCAD\", iteration=7, learning_rate=0.11, depth=11, window_size=6, alpha_type=\"delta\") # version passed\n",
    "# test_model(symbol=\"USDCAD\", iteration=8, learning_rate=0.11, depth=11, window_size=6, alpha_type=\"delta\", save_model=True) # version A passed\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# xxtest_model(symbol=\"NZDUSD\", iteration=111, learning_rate=0.21, depth=6, window_size=6, alpha_type=\"gamma\") # version A\n",
    "# test_model(symbol=\"NZDUSD\", iteration=77, learning_rate=0.21, depth=6, window_size=6, alpha_type=\"gamma\") # version A\n",
    "# test_model(symbol=\"NZDUSD\", iteration=9, learning_rate=0.25, depth=10, window_size=7, alpha_type=\"gamma\") # version A\n",
    "# test_model(symbol=\"NZDUSD\", iteration=11, learning_rate=0.25, depth=10, window_size=7, alpha_type=\"gamma\", save_model=True) # version A passed\n",
    "\n",
    "# test_model(symbol=\"NZDUSD\", iteration=1, learning_rate=0.3, depth=7, window_size=7, alpha_type=\"alpha\") # version \n",
    "# test_model(symbol=\"NZDUSD\", iteration=3, learning_rate=0.7, depth=11, window_size=7, alpha_type=\"gamma\") # version \n",
    "# test_model(symbol=\"NZDUSD\", iteration=11, learning_rate=0.2, depth=10, window_size=7, alpha_type=\"gamma\") # version  \n",
    "\n",
    "\n",
    "# test_model(symbol=\"NZDUSD\", iteration=17, learning_rate=0.29, depth=10, window_size=7, alpha_type=\"gamma\") # another version\n",
    "# test_model(symbol=\"NZDUSD\", iteration=21, learning_rate=0.21, depth=6, window_size=7, alpha_type=\"gamma\") # another version\n",
    "# test_model(symbol=\"NZDUSD\", iteration=3, learning_rate=0.15, depth=9, window_size=7, alpha_type=\"gamma\") # another version\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"GBPUSD\", iteration=3,  learning_rate=0.25, depth=11, window_size=6,  alpha_type=\"delta\") # version \n",
    "# test_model(symbol=\"GBPUSD\", iteration=17,  learning_rate=0.25, depth=11, window_size=9,  alpha_type=\"delta\") # another version \n",
    "# test_model(symbol=\"GBPUSD\", iteration=5,  learning_rate=0.15, depth=10, window_size=11, alpha_type=\"gamma\") # version A\n",
    "\n",
    "# test_model(symbol=\"GBPUSD\", iteration=7, learning_rate=0.19, depth=9, window_size=11, alpha_type=\"gamma\") # another version A \n",
    "# test_model(symbol=\"GBPUSD\", iteration=19, learning_rate=0.19, depth=9, window_size=11, alpha_type=\"gamma\", save_model=True) # another version A passed\n",
    "# test_model(symbol=\"GBPUSD\", iteration=9, learning_rate=0.21, depth=9, window_size=11, alpha_type=\"omega\") # another version A\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# xxtest_model(symbol=\"GBPJPY\", iteration=5, learning_rate=0.21, depth=9, window_size=7, alpha_type=\"alpha\") # version A \n",
    "# test_model(symbol=\"GBPJPY\", iteration=17, learning_rate=0.35, depth=11, window_size=6, alpha_type=\"alpha\") # version A alpha\n",
    "# test_model(symbol=\"GBPJPY\", iteration=8, learning_rate=0.17, depth=11, window_size=13, alpha_type=\"delta\") # version seems passed\n",
    "\n",
    "# test_model(symbol=\"GBPJPY\", iteration=8, learning_rate=0.17, depth=11, window_size=13, alpha_type=\"delta\") # version \n",
    "\n",
    "# test_model(symbol=\"GBPJPY\", iteration=3, learning_rate=0.35, depth=11, window_size=6, alpha_type=\"alpha\", save_model=True) # version passed close to A\n",
    "\n",
    "# test_model(symbol=\"GBPJPY\", iteration=7, learning_rate=0.35, depth=11, window_size=6, alpha_type=\"alpha\") # version A passed\n",
    "\n",
    "\n",
    "# test_model(symbol=\"GBPJPY\", iteration=7, learning_rate=0.15, depth=9, window_size=13, alpha_type=\"gamma\") # another version accuracy by net gains 54-57 \n",
    "# test_model(symbol=\"GBPJPY\", iteration=2, learning_rate=0.7, depth=13, window_size=6, alpha_type=\"alpha\") # another version \n",
    "# test_model(symbol=\"GBPJPY\", iteration=5, learning_rate=0.3, depth=10, window_size=7, alpha_type=\"alpha\") # another version \n",
    "# test_model(symbol=\"GBPJPY\", iteration=5, learning_rate=0.3, depth=10, window_size=7, alpha_type=\"alpha\") # another version \n",
    "# test_model(symbol=\"GBPJPY\", iteration=7, learning_rate=0.6, depth=12, window_size=11, alpha_type=\"gamma\") # another version \n",
    "# test_model(symbol=\"GBPJPY\", iteration=3, learning_rate=0.7, depth=12, window_size=6, alpha_type=\"alpha\") # another version close to A\n",
    "# test_model(symbol=\"GBPJPY\", iteration=3, learning_rate=0.19, depth=10, window_size=3, alpha_type=\"delta\") # another version \n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"GBPAUD\", iteration=9, learning_rate=0.15, depth=6, window_size=7)\n",
    "# test_model(symbol=\"GBPAUD\", iteration=5, learning_rate=0.15, depth=9, window_size=11) # another version\n",
    "# test_model(symbol=\"GBPAUD\", iteration=15, learning_rate=0.3, depth=9, window_size=7)  # another version\n",
    "# test_model(symbol=\"GBPAUD\", iteration=100, learning_rate=0.05, depth=6, window_size=6)# another version \n",
    "# xxtest_model(symbol=\"GBPAUD\", iteration=44, learning_rate=0.05, depth=6, window_size=9) # version A\n",
    "\n",
    "# test_model(symbol=\"GBPAUD\", iteration=100, learning_rate=0.05, depth=6, window_size=9) # version seems passed\n",
    "# test_model(symbol=\"GBPAUD\", iteration=12, learning_rate=0.07, depth=5, window_size=6,alpha_type=\"gamma\", save_model=True) # version A passed\n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"AUDJPY\", iteration=3, learning_rate=0.2, depth=9, window_size=6, alpha_type=\"omega\") # version omega\n",
    "\n",
    "# test_model(symbol=\"AUDJPY\", iteration=1, learning_rate=0.45, depth=11, window_size=6, alpha_type=\"beta\") # version \n",
    "# test_model(symbol=\"AUDJPY\", iteration=3, learning_rate=0.15, depth=11, window_size=7, alpha_type=\"beta\") # version \n",
    "\n",
    "# test_model(symbol=\"AUDJPY\", iteration=1, learning_rate=0.25, depth=11, window_size=7, alpha_type=\"delta\") # version A delta passed\n",
    "# test_model(symbol=\"AUDJPY\", iteration=1, learning_rate=0.2, depth=11, window_size=7, alpha_type=\"delta\", save_model=True) # version A delta passed\n",
    "\n",
    "# test_model(symbol=\"AUDJPY\", iteration=27, learning_rate=0.2, depth=10, window_size=11, alpha_type=\"omega\") # version omega\n",
    "\n",
    "# test_model(symbol=\"AUDJPY\", iteration=1, learning_rate=0.2, depth=11, window_size=9, alpha_type=\"alpha\") # version A\n",
    "# test_model(symbol=\"AUDJPY\", iteration=1, learning_rate=0.15, depth=13, window_size=7, alpha_type=\"alpha\") # another version\n",
    "# test_model(symbol=\"AUDJPY\", iteration=3, learning_rate=0.15, depth=12, window_size=7, alpha_type=\"alpha\") # another version\n",
    "\n",
    "\n",
    "# test_model(symbol=\"AUDJPY\", iteration=3, learning_rate=0.7, depth=12, window_size=6, alpha_type=\"gamma\") # another version\n",
    "# test_model(symbol=\"AUDJPY\", iteration=1, learning_rate=0.01, depth=9, window_size=9, alpha_type=\"alpha\") # another version\n",
    "# test_model(symbol=\"AUDJPY\", iteration=5, learning_rate=0.15, depth=13, window_size=7, alpha_type=\"alpha\") # another version\n",
    "\n",
    "\n",
    "# test_model(symbol=\"EURJPY\", iteration=10, learning_rate=0.7, depth=6, window_size=7, alpha_type=\"alpha\") # another version\n",
    "\n",
    "# xxtest_model(symbol=\"EURJPY\", iteration=13, learning_rate=0.2, depth=11, window_size=5, alpha_type=\"gamma\") # version A\n",
    "# test_model(symbol=\"EURJPY\", iteration=11, learning_rate=0.2, depth=11, window_size=5, alpha_type=\"gamma\") # version A \n",
    "# test_model(symbol=\"EURJPY\", iteration=5, learning_rate=0.2, depth=11, window_size=5, alpha_type=\"gamma\", save_model=True) # version A passed\n",
    "\n",
    "# test_model(symbol=\"EURJPY\", iteration=5, learning_rate=0.2, depth=9, window_size=7, alpha_type=\"delta\") # another version\n",
    "\n",
    "# test_model(symbol=\"EURJPY\", iteration=7, learning_rate=0.5, depth=5, window_size=11, alpha_type=\"gamma\") # another version\n",
    "\n",
    "\n",
    "# test_model(symbol=\"AUDNZD\", iteration=1, learning_rate=0.5, depth=11, window_size=12, alpha_type=\"omega\") # another version\n",
    "# test_model(symbol=\"AUDNZD\", iteration=3, learning_rate=0.65, depth=10, window_size=12, alpha_type=\"omega\") # version A\n",
    "\n",
    "# test_model(symbol=\"AUDNZD\", iteration=3, learning_rate=0.65, depth=10, window_size=12, alpha_type=\"beta\")   # version A\n",
    "# test_model(symbol=\"AUDNZD\", iteration=3, learning_rate=0.65, depth=10, window_size=12, alpha_type=\"beta\")   # version A\n",
    "\n",
    "# test_model(symbol=\"AUDNZD\", iteration=13, learning_rate=0.5, depth=9, window_size=9, alpha_type=\"gamma\") # version seems passed\n",
    "# test_model(symbol=\"AUDNZD\", iteration=5, learning_rate=0.15, depth=11, window_size=7, alpha_type=\"beta\") # version A seems passed\n",
    "# test_model(symbol=\"AUDNZD\", iteration=9, learning_rate=0.03, depth=10, window_size=6, alpha_type=\"beta\", save_model=True) # version A passed\n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"GBPNZD\", iteration=3, learning_rate=0.2, depth=10, window_size=7, alpha_type=\"omega\")\n",
    "# test_model(symbol=\"GBPNZD\", iteration=3, learning_rate=0.2, depth=9, window_size=6, alpha_type=\"gamma\")\n",
    "# xxtest_model(symbol=\"GBPNZD\", iteration=13, learning_rate=0.25, depth=10, window_size=7, alpha_type=\"beta\") # version A\n",
    "# test_model(symbol=\"GBPNZD\", iteration=15, learning_rate=0.25, depth=10, window_size=7, alpha_type=\"beta\") # version A seems passed\n",
    "# test_model(symbol=\"GBPNZD\", iteration=12, learning_rate=0.25, depth=10, window_size=7, alpha_type=\"beta\", save_model=True) # version A  passed\n",
    "# test_model(symbol=\"GBPNZD\", iteration=33, learning_rate=0.15, depth=12, window_size=5, alpha_type=\"beta\") # version A\n",
    "\n",
    "\n",
    "# test_model(symbol=\"AUDUSD\", iteration=5, learning_rate=0.27, depth=10, window_size=9, alpha_type=\"gamma\") # version \n",
    "# test_model(symbol=\"AUDUSD\", iteration=5, learning_rate=0.15, depth=12, window_size=5, alpha_type=\"beta\") # version A\n",
    "# test_model(symbol=\"AUDUSD\", iteration=33, learning_rate=0.15, depth=12, window_size=5, alpha_type=\"beta\") # version A seems passed\n",
    "# test_model(symbol=\"AUDUSD\", iteration=11, learning_rate=0.15, depth=10, window_size=6, alpha_type=\"delta\") # version \n",
    "# test_model(symbol=\"AUDUSD\", iteration=3, learning_rate=0.1, depth=10, window_size=11, alpha_type=\"delta\") # version seems passed\n",
    "# test_model(symbol=\"AUDUSD\", iteration=7, learning_rate=0.27, depth=10, window_size=9, alpha_type=\"gamma\") # version  \n",
    "# test_model(symbol=\"AUDUSD\", iteration=23, learning_rate=0.25, depth=8, window_size=7, alpha_type=\"gamma\", save_model=True) # version A passed \n",
    "\n",
    "# xxtest_model(symbol=\"AUDUSD\", iteration=21, learning_rate=0.21, depth=9, window_size=9, alpha_type=\"alpha\") # version \n",
    "# test_model(symbol=\"AUDUSD\", iteration=14, learning_rate=0.21, depth=7, window_size=9, alpha_type=\"beta\", save_model=True) # version A seems passed\n",
    "# xx test_model(symbol=\"AUDUSD\", iteration=5, learning_rate=0.15, depth=11, window_size=7, alpha_type=\"omega\") # version \n",
    "\n",
    "\n",
    "# test_model(symbol=\"NZDJPY\", iteration=3, learning_rate=0.1, depth=12, window_size=7, alpha_type=\"beta\") # version  \n",
    "\n",
    "# test_model(symbol=\"NZDJPY\", iteration=5, learning_rate=0.35, depth=11, window_size=7, alpha_type=\"beta\") # version  A\n",
    "# test_model(symbol=\"NZDJPY\", iteration=11, learning_rate=0.35, depth=11, window_size=7, alpha_type=\"beta\", save_model=True) # version  A passed\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# test_model(symbol=\"USDCHF\", iteration=3, learning_rate=0.25, depth=12, window_size=6, alpha_type=\"gamma\") # version passed close to A\n",
    "# test_model(symbol=\"USDCHF\", iteration=17, learning_rate=0.15, depth=7, window_size=11, alpha_type=\"beta\") # version passed\n",
    "# test_model(symbol=\"USDCHF\", iteration=7, learning_rate=0.22, depth=8, save_model=False, window_size=6, alpha_type=\"ha\" ) # version passed\n",
    "# test_model(symbol=\"USDCHF\", iteration=6, learning_rate=0.22, depth=8, window_size=6, alpha_type=\"ha\", save_model=True) # version passed close to A\n",
    "# xx test_model(symbol=\"USDCHF\", iteration=19, learning_rate=0.19, depth=5, window_size=5, alpha_type=\"rsi\") # version \n",
    "\n",
    "\n",
    "# ___ In progress\n",
    "# test_model(symbol=\"GBPCAD\", iteration=6, learning_rate=0.22, depth=8, window_size=6, alpha_type=\"ha\", save_model=False) # version \n",
    "\n",
    "\n",
    "# test_model(symbol=\"GBPCAD\", iteration=13, learning_rate=0.25, depth=8, window_size=7, alpha_type=\"alpha\") # version passed A\n",
    "# test_model(symbol=\"GBPCAD\", iteration=15, learning_rate=0.25, depth=8, window_size=7, alpha_type=\"alpha\") # version passed close to A\n",
    "\n",
    "# test_model(symbol=\"GBPCAD\", iteration=35, learning_rate=0.25, depth=8, window_size=5, alpha_type=\"gamma\") # version \n",
    "\n",
    "\n",
    "# test_model(symbol=\"AUDCAD\", iteration=23, learning_rate=0.25, depth=8, window_size=5, alpha_type=\"gamma\" ) # # version passed A\n",
    "\n",
    "# test_model(symbol=\"CHFJPY\", iteration=11, learning_rate=0.75, depth=8, window_size=5, alpha_type=\"alpha\" ) # version close to A\n",
    "\n",
    "# test_model(symbol=\"CHFJPY\", iteration=21, learning_rate=0.75, depth=8, window_size=5, alpha_type=\"alpha\" ) # version A\n",
    "\n",
    "\n",
    "# test_model(symbol=\"NZDCHF\", iteration=11, learning_rate=0.77, depth=7, window_size=5, alpha_type=\"beta\") # version close to A\n",
    "# \n",
    "# test_model(symbol=\"NZDCHF\", iteration=22, learning_rate=0.77, depth=7, window_size=5, alpha_type=\"beta\") # version A passed \n",
    "\n",
    "# test_model(symbol=\"NZDCHF\", iteration=4, learning_rate=0.5, depth=9, window_size=7, alpha_type=\"gamma\") # version \n",
    "\n",
    "# test_model(symbol=\"NZDCHF\", iteration=11, learning_rate=0.15, depth=8, window_size=5, alpha_type=\"gamma\") # version \n",
    "# test_model(symbol=\"EURGBP\", iteration=14, learning_rate=0.18, depth=9, window_size=6, alpha_type=\"omega\") # version A passed\n",
    "# test_model(symbol=\"EURGBP\", iteration=15, learning_rate=0.18, depth=9, window_size=6, alpha_type=\"omega\") # version A passed\n",
    "# test_model(symbol=\"EURGBP\", iteration=7, learning_rate=0.77, depth=8, window_size=7, alpha_type=\"gamma\") # version close to A\n",
    "\n",
    "# ____ END\n",
    "# ____________________________________________\n",
    "\n",
    "# test_model(symbol=\"AUDCAD\", iteration=23, learning_rate=0.25, depth=8, window_size=5, alpha_type=\"gamma\", save_model=True) # # version passed A\n",
    "\n",
    "# test_model(symbol=\"CHFJPY\", iteration=21, learning_rate=0.75, depth=8, window_size=5, alpha_type=\"alpha\" , save_model=True) # version A\n",
    "\n",
    "# test_model(symbol=\"GBPCAD\", iteration=13, learning_rate=0.25, depth=8, window_size=7, alpha_type=\"alpha\", save_model=True) # version passed A\n",
    "\n",
    "\n",
    "# test_model(symbol=\"NZDCHF\", iteration=22, learning_rate=0.77, depth=7, window_size=5, alpha_type=\"beta\", save_model=True) # version A passed \n",
    "\n",
    "\n",
    "# test_model(symbol=\"EURGBP\", iteration=14, learning_rate=0.18, depth=9, window_size=6, alpha_type=\"omega\", save_model=True) # version A passed\n",
    "\n",
    "\n",
    "# test_model(symbol=\"CADCHF\", iteration=15, learning_rate=0.18, depth=8, window_size=6, alpha_type=\"beta\") # version close to A \n",
    "\n",
    "# test_model(symbol=\"CADCHF\", iteration=14, learning_rate=0.20, depth=7, window_size=7, alpha_type=\"beta\") # version close to A\n",
    "\n",
    "# test_model(symbol=\"CADCHF\", iteration=14, learning_rate=0.20, depth=7, window_size=7, alpha_type=\"omega\", save_model=True) # version A passed\n",
    "\n",
    "\n",
    "# test_model(symbol=\"AUDCHF\", iteration=25, learning_rate=0.25, depth=7, window_size=11, alpha_type=\"delta\", save_model=True) # version A passed \n",
    "\n",
    "# test_model(symbol=\"AUDCHF\", iteration=22, learning_rate=0.25, depth=7, window_size=11, alpha_type=\"delta\") # version close to A\n",
    "\n",
    "# test_model(symbol=\"AUDCHF\", iteration=27, learning_rate=0.66, depth=8, window_size=6, alpha_type=\"alpha\") # version close to A \n",
    "\n",
    "\n",
    "# test_model(symbol=\"GBPCHF\", iteration=13, learning_rate=0.49, depth=7, window_size=5, alpha_type=\"beta\") # version close to A\n",
    "\n",
    "# test_model(symbol=\"GBPCHF\", iteration=13, learning_rate=0.42, depth=7, window_size=5, alpha_type=\"beta\", save_model=True) # version A passed\n",
    "\n",
    "# test_model(symbol=\"GBPCHF\", iteration=12, learning_rate=0.42, depth=7, window_size=5, alpha_type=\"beta\") # version \n",
    "\n",
    "\n",
    "# test_model(symbol=\"EURCHF\", iteration=19, learning_rate=0.18, depth=7, window_size=5, alpha_type=\"omega\") # version close to A\n",
    "# test_model(symbol=\"EURCHF\", iteration=20, learning_rate=0.18, depth=7, window_size=5, alpha_type=\"omega\", save_model=True) # version A passed\n",
    "# test_model(symbol=\"EURCHF\", iteration=12, learning_rate=0.44, depth=7, window_size=5, alpha_type=\"beta\") # version close to A\n",
    "\n",
    "# test_model( symbol=\"CADJPY\", iteration=7, learning_rate=0.38, depth=9, window_size=11, alpha_type=\"delta\", save_model=True) # version A passed\n",
    "\n",
    "# test_model( symbol=\"CADJPY\", iteration=7, learning_rate=0.71, depth=12, window_size=6, alpha_type=\"beta\" ) version close to A\n",
    "\n",
    "# test_model( symbol=\"CADJPY\", iteration=47, learning_rate=0.47, depth=9, window_size=5, alpha_type=\"beta\" ) # version A passed\n",
    "\n",
    "# test_model( symbol=\"CADJPY\", iteration=7, learning_rate=0.38, depth=9, window_size=11, alpha_type=\"delta\" ) # version A passed\n",
    "\n",
    "# test_model( symbol=\"CADJPY\", iteration=110, learning_rate=0.27, depth=6, window_size=11, alpha_type=\"delta\" )\n",
    "\n",
    "\n",
    "# test_model( symbol=\"NZDCAD\", iteration=7, learning_rate=0.2, depth=12, window_size=9, alpha_type=\"omega\" )\n",
    "# test_model( symbol=\"NZDCAD\", iteration=7, learning_rate=0.2, depth=12, window_size=9, alpha_type=\"gamma\" ) # version close to A \n",
    "\n",
    "# test_model( symbol=\"NZDCAD\", iteration=6, learning_rate=0.35, depth=11, window_size=6, alpha_type=\"gamma\" ) # version close to A\n",
    "# test_model( symbol=\"NZDCAD\", iteration=7, learning_rate=0.36, depth=11, window_size=6, alpha_type=\"gamma\" ) # version close to A\n",
    "\n",
    "# test_model( symbol=\"NZDCAD\", iteration=7, learning_rate=0.68, depth=11, window_size=6, alpha_type=\"gamma\", save_model=True ) # version A passed\n",
    "\n",
    "\n",
    "# test_model( symbol=\"GBPJPY\", iteration=27, learning_rate=0.27, depth=7, window_size=8, alpha_type=\"beta\") # version A passed \n",
    "# test_model( symbol=\"GBPJPY\", iteration=10, learning_rate=0.1, depth=10, window_size=5, alpha_type=\"lambda\", save_model=True) # version A passed\n",
    "\n",
    "\n",
    "\n",
    "# __________\n",
    "# experimental \n",
    "# test_model(symbol=\"USDCHF\", iteration=4, learning_rate=0.18, depth=8, window_size=5, alpha_type=\"beta\") # version passed\n",
    "# test_model(symbol=\"USDCHF\", iteration=3, learning_rate=0.78, depth=13, window_size=7, alpha_type=\"omega\") # version passed close to A\n",
    "# test_model(symbol=\"USDCHF\", iteration=3, learning_rate=0.78, depth=11, window_size=5, alpha_type=\"omega\") # version passed close to A\n",
    "# test_model(symbol=\"USDCHF\", iteration=7, learning_rate=0.15, depth=8, window_size=7, alpha_type=\"ha\") # version passed close to A\n",
    "# test_model(symbol=\"USDCHF\", iteration=3, learning_rate=0.58, depth=11, window_size=5, alpha_type=\"omega\") # version passed \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c655aeaf-eb55-4558-8540-bc48774763f5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\n",
      "45\n"
     ]
    }
   ],
   "source": [
    "step = 2\n",
    "cluster = 22\n",
    "end, start = 0, 0\n",
    "\n",
    "for i in range(step): # 1-> 22 + 1 # 2    \n",
    "        end   += cluster\n",
    "\n",
    "end += 1\n",
    "start = end - cluster\n",
    "\n",
    "print(start)\n",
    "print(end)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4aef7b78-e7e2-464e-bf6d-8bd837beee7e",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/pandas/core/algorithms.py:1601: FutureWarning: Comparison of Timestamp with datetime.date is deprecated in order to match the standard library behavior.  In a future version these will be considered non-comparable.Use 'ts == pd.Timestamp(date)' or 'ts.date() == date' instead.\n",
      "  return arr.searchsorted(value, side=side, sorter=sorter)\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:35: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:35: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:42: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:42: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/pandas/core/generic.py:6392: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return self._update_inplace(result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USDJPY\n",
      "0:\tlearn: 70.4186142\ttotal: 554ms\tremaining: 1.11s\n",
      "2:\tlearn: 65.9572596\ttotal: 1.58s\tremaining: 0us\n",
      "Mean Squared Error: 6107.159143442258\n",
      "MAY\n",
      "Sharpe Ratio: 0.37\n",
      "['-1+1', -1, '-1+1', '-1+1', -1, '+1-1', 1, 1, '+1-1', 1, 1, '-1+1', -1, 1, 1, 1, '+1-1', 1, 1, 1, '+1-1', '+1-1', -1]\n",
      "Pips On Profit Side was :  1073.9999999999982\n",
      "Pips On Loss Side was :  353.3000000000044\n",
      "accuracy by days 60.86956521739131\n",
      "accuracy_by_net_gains  75.24696980312451\n",
      "net profit  720.6999999999938\n",
      "\n",
      "accuracy by net gains on previous test data  62.900529525481666\n",
      "accuracy on all test points excluding last 22 points  56.68449197860963\n",
      "APRIL\n",
      "Sharpe Ratio: 0.64\n",
      "[-1, 1, 1, 1, '+1-1', 1, 1, 1, '+1-1', '+1-1', -1, 1]\n",
      "Pips On Profit Side was :  969.199999999995\n",
      "Pips On Loss Side was :  166.19999999999777\n",
      "accuracy by days 75.0\n",
      "accuracy_by_net_gains  85.36198696494638\n",
      "net profit  802.9999999999973\n",
      "\n",
      "accuracy by net gains on previous test data  61.7569193742479\n",
      "accuracy on all test points excluding last 22 points  56.451612903225815\n",
      "MARCH\n",
      "Sharpe Ratio: 0.13\n",
      "[1, '+1-1', 1, '+1-1', '+1-1', 1, 1, 1, 1, 1, 1, 1, 1, '+1-1', '+1-1', '+1-1', -1, '+1-1', 1, '-1+1', '+1-1', 1, '+1-1']\n",
      "Pips On Profit Side was :  621.2000000000046\n",
      "Pips On Loss Side was :  434.89999999999895\n",
      "accuracy by days 56.52173913043478\n",
      "accuracy_by_net_gains  58.82018748224623\n",
      "net profit  186.30000000000564\n",
      "\n",
      "accuracy by net gains on previous test data  57.897843974073574\n",
      "accuracy on all test points excluding last 22 points  55.319148936170215\n",
      "Iterations:  3\n",
      "Learning rate:  0.2\n",
      "Depth:  12\n",
      "FEB\n",
      "Sharpe Ratio: 0.31\n",
      "[1, '+1-1', 1, 1, '+1-1', '+1-1', 1, '+1-1', '+1-1', 1, 1, '+1-1', 1, 1, '+1-1', 1, 1, '+1-1', '+1-1', 1, -1, 1, 1]\n",
      "Pips On Profit Side was :  751.3000000000005\n",
      "Pips On Loss Side was :  301.00000000000193\n",
      "accuracy by days 60.86956521739131\n",
      "accuracy_by_net_gains  71.39598973676696\n",
      "net profit  450.2999999999986\n",
      "\n",
      "accuracy by net gains on previous test data  56.14431940281626\n",
      "accuracy on all test points excluding last 22 points  55.08474576271186\n",
      "JAN\n",
      "Sharpe Ratio: 0.12\n",
      "['+1-1', '-1+1', 1, '+1-1', '+1-1', 1, 1, 1, '+1-1', '+1-1', '-1+1', 1, '+1-1', '+1-1', 1, 1, 1, '+1-1', '+1-1', '+1-1', '+1-1', 1, '+1-1']\n",
      "Pips On Profit Side was :  743.1999999999988\n",
      "Pips On Loss Side was :  542.0999999999964\n",
      "accuracy by days 39.130434782608695\n",
      "accuracy_by_net_gains  57.8230763245936\n",
      "net profit  201.1000000000024\n",
      "\n",
      "accuracy by net gains on previous test data  55.51523947750364\n",
      "accuracy on all test points excluding last 22 points  57.89473684210527\n",
      "DEC\n",
      "Sharpe Ratio: 0.01\n",
      "[1, '+1-1', '+1-1', 1, 1, 1, '+1-1', '+1-1', '+1-1', 1, 1, -1, 1, '+1-1', 1, '+1-1', 1, '+1-1', '+1-1', '+1-1', '+1-1', 1, 1]\n",
      "Pips On Profit Side was :  1015.2000000000044\n",
      "Pips On Loss Side was :  976.3000000000034\n",
      "accuracy by days 52.17391304347826\n",
      "accuracy_by_net_gains  50.976650765754485\n",
      "net profit  38.900000000001\n",
      "\n",
      "accuracy by net gains on previous test data  59.355269236692145\n",
      "accuracy on all test points excluding last 22 points  61.111111111111114\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fb624f9b-8aee-47d1-b74a-255d3fd57b90",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "\n",
    "# Results Comparison "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "694f5002-3e59-43ba-871c-a122981a4a0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "results_dict = {}\n",
    "day_features, hour4_features = load_features_files()\n",
    "forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY'\n",
    "        ]\n",
    "\n",
    "symbol_hyperparameter = [\n",
    " 'AUDJPY', 'AUDUSD', 'AUDCAD', 'AUDCHF', 'AUDNZD',\n",
    " 'CADJPY', \n",
    " 'EURAUD', 'EURCAD', 'EURUSD', 'EURGBP', 'EURNZD',\n",
    " 'GBPCAD', 'GBPCHF', 'GBPUSD', 'GBPNZD',\n",
    " 'NZDCHF', 'NZDJPY',\n",
    " 'USDCHF', 'USDJPY', 'USDCAD',\n",
    " 'CADCHF', \n",
    " 'NZDCAD', 'NZDUSD']\n",
    "\n",
    "# symbols = ['USDCAD']\n",
    "\n",
    "for symbol in forex_pairs:\n",
    "    \n",
    "    \n",
    "    day_data, hour4_data = get_day_hour4_features(symbol, day_features, hour4_features)\n",
    "\n",
    "    X_train_scaled, X_test_scaled, y_train, y_test = get_features_transformed(symbol, day_data, hour4_data)\n",
    "    \n",
    "    print(symbol)\n",
    "    # model = load_model(symbol, path = f\"forex_models_custom_hyperparam/{symbol}_model_pips_change\")\n",
    "    # model = load_model(symbol, path =f'forex_models_low_lr_few_h4features_300 iteration/{symbol}_model_pips_change')\n",
    "    # model = load_model(symbol, path = f\"forex_models_regularized\")\n",
    "    model = load_model(symbol, path = f\"forex_models_regularized_h4_lag_3\")\n",
    "    \n",
    "    \n",
    "    #   \n",
    "\n",
    "    # if symbol in symbol_hyperparameter: \n",
    "      \n",
    "#        symbol_parameters = load_parameters(symbol)\n",
    "#        iteration, learning_rate, depth = symbol_parameters['Iterations'], symbol_parameters['Lr'], symbol_parameters['depth']\n",
    "#        parameters = [ iteration, learning_rate, depth ] \n",
    "#        model = train_model(X_train_scaled, X_test_scaled, y_train, y_test, parameters=parameters)\n",
    "    \n",
    "#     else:\n",
    "#       model = train_model(X_train_scaled, X_test_scaled, y_train, y_test)\n",
    "\n",
    "    # model = load_model(symbol)\n",
    "    gains , accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled,  y_test)\n",
    "    \n",
    "     # 2  # 1 --> May 2024 , # 2 --> April 2024\n",
    "    # step = 1 # ____          May \n",
    "    # step = 5 \n",
    "    \n",
    "    # step = 4 # February\n",
    "\n",
    "    # step = 5 # January\n",
    "\n",
    "    step = 6 # December\n",
    "    # step = 7 # November\n",
    "    # step = 8 # October\n",
    "    \n",
    "    month_key = {'8': 'oct', '7': 'nov', '6':'dec', '5': 'jan', '4':'feb', '3':'mar', '2':'april', '1':'may' }\n",
    "    month_name = 'may' if step == 1 else 'april'\n",
    "    \n",
    "    try: month_name = month_key[str(step)]\n",
    "    except : pass\n",
    "    \n",
    "    cluster = 22 # possible days of trading in a year data ends at 31 May\n",
    "    gains, accuracy = evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= [step, cluster] ) # custom_sample= None, last_cluster = None  \n",
    "    \n",
    "    \n",
    "    if step == 3 : month_name = 'march' \n",
    "    if step == 4 : month_name = 'feb'\n",
    "    if step == 5 : month_name = 'jan'\n",
    "    net_gains = gains['net_gains']\n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    # results_dict[symbol] = { 'net_gains': int(net_gains), 'accuracy': int(accuracy), 'month': month_name, \n",
    "    #                          'accuracy_by_net_gains': int(accuracy_by_net_gains)  }\n",
    "\n",
    "    net_gains = gains['net_gains']\n",
    "    \n",
    "    accuracy_by_net_gains = gains['accuracy_by_net_gains']\n",
    "    results_dict[symbol] = { 'net_gains': int(net_gains), 'accuracy': int(accuracy), 'month': month_name, \n",
    "                             'accuracy_by_net_gains': int(accuracy_by_net_gains)  }\n",
    "    # save_model(symbol, model)\n",
    "    \n",
    "    # accuracy_by_net_gains = gains['accuracy_by_net_gains_before']\n",
    "    accuracy_by_net_gains_before = gains['accuracy_by_net_gains_before']\n",
    "\n",
    "    print('net profit ', net_gains)\n",
    "    # print('accuracy_by_net_gains \\n',accuracy_by_net_gains )\n",
    "    print('\\naccuracy by net gains on previous test data ', accuracy_by_net_gains_before)\n",
    "    print(\"accuracy on all test points excluding last 22 points \", accuracy )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb44e9f4-65cd-48ae-b9aa-7a97a56e15e4",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Version 2\n",
    "* Temporal features True (Month, Day of week, Week Of Month)\n",
    "* Hour 4 Features Lag by 7\n",
    "* Day Features (All included) Lag by 9\n",
    "* Starting Date June 2020 Training Till September 2023\n",
    "* Ending Date May 2024\n",
    "* Hour 4 Features 'relative range', 'candle type', 'heikin ashi' \n",
    "* Hyperparameters Regularized\n",
    "\n",
    "\n",
    "Version 1 Regularized On Two Months April May On Two Conditions :\n",
    " \n",
    " 1- hyper parameters chosen should give positive gains for any currency pairs for those two months\n",
    " \n",
    " 2- Previous months accuracy should be 50% and net gains accuracy close to 50% or 52%\n",
    "\n",
    "Results :- (pips)\n",
    "* ✅October 2167\n",
    "* ✅ November 2498\n",
    "* ❌ December -3080\n",
    "* ✅ January 4844\n",
    "* ✅ February 1288\n",
    "* ✅ March 1883\n",
    "\n",
    "📉 Not all models were in positive gains in previous month despite all performing good on april may with positive gains\n",
    "\n",
    "⭐ But parameters performing good on two different months gave them ability to keep majority of currency pairs model in positive gain\n",
    "\n",
    "💢 December is also not recommended for trading by trading community so we might ignore that month outcomes \n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d890b2b-6cdf-4f86-b862-28a4183811d3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "355e01a6-8b84-4995-b4de-3825a46eedeb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d54fd1b-83a4-4dff-a8c7-ecb2633d2521",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e44cfab5-2090-49dc-97c0-7b1d71e3bbd5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# For December On Apri-May Month Regularized Data\n",
    "# print(results_dict)\n",
    "\n",
    "\n",
    "# Calculate the final sum of net gains\n",
    "final_sum_net_gain = sum(info['net_gains'] for info in results_dict.values())\n",
    "\n",
    "# print(\"Final sum of net gains December: \", final_sum_net_gain)\n",
    "\n",
    "\n",
    "# Create a list of tuples (symbol, net_gain) and sort it by net_gain in descending order\n",
    "sorted_list = sorted(results_dict.items(), key=lambda x: x[1]['net_gains'], reverse=False) \n",
    "# reverse = False ascending order\n",
    "# reverse = True descending order\n",
    "\n",
    "\n",
    "# Convert the sorted list back to a dictionary if needed\n",
    "sorted_dict = {symbol: info for symbol, info in sorted_list}\n",
    "\n",
    "# final_sum_net_gain = (  dict(symbol, symbol['pips_loss']) for symbol in pl_info.values())\n",
    "\n",
    "loss_list = [ (symbol, value['net_gains'])  for symbol, value in sorted_dict.items() ]\n",
    "\n",
    "\n",
    "print(\"Most Losses In December: \", loss_list )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be510e95-e455-4103-b559-c628c0c5dbb3",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Hyperparameters Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "13c97177-788d-4d92-8e6f-8705e0fe78df",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "parameters = {\n",
    "    'default':{\n",
    "        'Iterations': 300,\n",
    "        'Lr': 0.01,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 6\n",
    "    },\n",
    "    \n",
    "    'NZDJPY': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.001,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 6\n",
    "    },\n",
    "    'AUDJPY': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.01,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 6\n",
    "    },\n",
    "    'USDJPY': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.01,\n",
    "        'depth': 6\n",
    "    },\n",
    "    'CADJPY': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.01,\n",
    "        'depth': 6\n",
    "    },\n",
    "    'NZDCAD': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 114\n",
    "    },\n",
    "    'CADCHF': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 158\n",
    "    },\n",
    "    'USDCHF2': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 50,\n",
    "        'net pips profit': 179\n",
    "    },\n",
    "    'EURAUD': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 56,\n",
    "        'net pips profit': 306\n",
    "    },\n",
    "    'EURNZD3': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 278\n",
    "    },\n",
    "    'GBPNZD': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 50,\n",
    "        'net pips profit': 224\n",
    "    },\n",
    "    'NZDCHF2': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 50,\n",
    "        'net pips profit': 140\n",
    "    },\n",
    "    'EURUSD3': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 56,\n",
    "        'net pips profit': 306\n",
    "    },\n",
    "    'GBPCAD': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.001,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 54,\n",
    "        'net pips profit': 151\n",
    "    },\n",
    "    'GBPUSD2': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.001,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 217\n",
    "    },\n",
    "    'EURGBP': {\n",
    "        'Lr': 0.001 ,\n",
    "        'Iterations': 188,\n",
    "        'depth': 7,\n",
    "        'accuracy' :52,\n",
    "        'pips net profit' : 97\n",
    "    },\n",
    "    'EURCAD2': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 51,\n",
    "        'pips net profit':  228\n",
    "    }, \n",
    "    'GBPCHF': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 53,\n",
    "        'pips net profit':  241\n",
    "    }, \n",
    "    'NZDUSD': {\n",
    "        'Iterations': 5, \n",
    "        'Lr': 0.77, \n",
    "        'depth': 7,\n",
    "        'accuracy': 54\n",
    "    }, \n",
    "    'EURUSD2': {\n",
    "        'Lr':0.025,\n",
    "        'Iterations': 450, \n",
    "        'depth': 5,\n",
    "        'accuracy': 53\n",
    "    },\n",
    "    'EURNZD2':{\n",
    "     'Iterations': 29, \n",
    "     'Lr': 0.63, \n",
    "     'depth': 7,\n",
    "     'accuracy': 50\n",
    "    }, \n",
    "    'GBPUSD': {\n",
    "     'Iterations': 13, \n",
    "     'Lr': 0.77, \n",
    "     'depth': 7,\n",
    "     'accuracy': 53\n",
    "    },\n",
    "    'GBPAUD': {\n",
    "    'Iterations': 13,\n",
    "    'Lr': 0.77,\n",
    "    'depth': 8,\n",
    "    'accuracy': 55       \n",
    "    },\n",
    "    'AUDUSD2': {\n",
    "    'Iterations': 5,\n",
    "    'Lr': 0.7,\n",
    "    'depth': 7,\n",
    "    'accuracy': 50       \n",
    "    },\n",
    "    'AUDCHF': {\n",
    "    'Iterations': 7,\n",
    "    'Lr': 0.71,\n",
    "    'depth' : 7\n",
    "    }, \n",
    "    'AUDCAD':{\n",
    "    'Iterations' : 9,\n",
    "    'Lr': 0.67,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'AUDNZD':{\n",
    "    'Iterations': 15,\n",
    "    'Lr': 0.68,\n",
    "    'depth': 7 \n",
    "    }, \n",
    "    'EURCAD':{ \n",
    "    'Iterations': 20,\n",
    "    'Lr': 0.039,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'NZDCHF' : {\n",
    "    'Iterations': 27,\n",
    "    'Lr' : 0.35,\n",
    "    'depth' : 7\n",
    "    },\n",
    "    'USDCAD': {\n",
    "    'Iterations': 20,\n",
    "    'Lr': 0.039,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'USDCHF':{\n",
    "    'Iterations': 188,\n",
    "    'Lr': 0.67,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'NZDCAD':{\n",
    "    'Iterations': 17,\n",
    "    'Lr': 0.38,\n",
    "    'depth': 7\n",
    "    }, \n",
    "    'AUDUSD' :{\n",
    "    'Iterations': 11 ,\n",
    "    'Lr': 0.211,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'EURUSD':{\n",
    "    'Iterations': 23,\n",
    "    'Lr': 0.51,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'EURNZD':{\n",
    "    'Iterations': 188,\n",
    "    'Lr' :0.55,\n",
    "    'depth': 7\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c35e8f12-3d27-4ebc-a977-2af709a2806a",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Python Code To Load Train And Evaluate Forex Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "37ba94b2-df0e-480e-8618-2b33a4b784b2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "from sklearn.preprocessing import RobustScaler, LabelEncoder, StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import classification_report\n",
    "from catboost import CatBoostClassifier\n",
    "from catboost import Pool\n",
    "from catboost import CatBoostRegressor\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split, RandomizedSearchCV\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from scipy.stats import randint\n",
    "\n",
    "\n",
    "def get_features_transformed(symbol, day_data, hour4_data, date_resume = '2020-06-01', X_test_re = False, temporal=False, window_size = 9, alpha_type= \"gamma\"):\n",
    "    \n",
    "    \n",
    "    # date_resume\n",
    "    # split_date = pd.to_datetime('2020-06-01') \n",
    "    \n",
    "    split_date = pd.to_datetime(date_resume)\n",
    "\n",
    "    day_data = day_data.loc[split_date:]\n",
    "\n",
    "\n",
    "    # split_date = pd.to_datetime('2020-06-01')\n",
    "\n",
    "    hour4_data = hour4_data.loc[split_date:]\n",
    "\n",
    "\n",
    "    day_data['pips_change'] = np.subtract( day_data['Close'].to_numpy(), day_data['Close'].shift(1).to_numpy() )\n",
    "    \n",
    "    if 'JPY' in symbol: pips_change = day_data['pips_change'] * 10 ** 2 \n",
    "    \n",
    "    \n",
    "    else :  pips_change = day_data['pips_change'] * 10 ** 4 \n",
    "    \n",
    "    day_data['Target'] = pips_change\n",
    "    \n",
    "    \n",
    "    # features =  [f'candle_type_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'rsi_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'Price_Range_T-{i}' for i in range(1, window_size + 1)  ] +\\\n",
    "    #            [f'rsi_crossover_slow_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'rsi_crossover_fast_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'harmonic_mean_low_T-{i}' for i in range(1, window_size + 1)  ] +\\\n",
    "    #            [f'harmonic_mean_high_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'slow_harmonic_mean_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'fast_harmonic_mean_T-{i}' for i in range(1, window_size + 1) ]\n",
    "             \n",
    "    # features = [f'Day_of_Week_T-{i}' for i in range(1, window_size + 1)] + [f'Week_of_Month_T-{i}' for i in range(1, window_size + 1)] +\\\n",
    "    #            [f'Month_T-{i}' for i in range(1, window_size + 1)]  +\\\n",
    "    #            [f'High_T-{i}' for i in range(1, window_size + 1) ] +  [f'Low_T-{i}'  for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'STDEV_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'Upper_Band_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'Lower_Band_T-{i}' for i in range(1, window_size + 1) ]+ \\\n",
    "    #            [f'candle_type_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'rsi_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'Price_Range_T-{i}' for i in range(1, window_size + 1)  ] +\\\n",
    "    #            [f'Median_Price_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'rsi_crossover_slow_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'rsi_crossover_fast_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'harmonic_mean_low_T-{i}' for i in range(1, window_size + 1)  ] +\\\n",
    "    #            [f'harmonic_mean_high_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'slow_harmonic_mean_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'fast_harmonic_mean_T-{i}' for i in range(1, window_size + 1) ] \n",
    "             \n",
    "             \n",
    "    # features = [f'Day_of_Week_T-{i}' for i in range(1, window_size + 1)] + [f'Week_of_Month_T-{i}' for i in range(1, window_size + 1)] +\\\n",
    "    #            [f'Month_T-{i}' for i in range(1, window_size + 1)]  +\\\n",
    "    #            [f'High_T-{i}' for i in range(1, window_size + 1) ] +  [f'Low_T-{i}'  for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'STDEV_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'RSI_T-{i}' for i in range(1, window_size + 1)   ] + \\\n",
    "    #            [f'Upper_Band_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'Lower_Band_T-{i}' for i in range(1, window_size + 1) ]+ \\\n",
    "    #            [f'candle_type_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'heikin_ashi_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'Price_Range_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'Median_Price_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'ema_3_T-{i}' for i in range(1, window_size + 1) ]  +\\\n",
    "    #            [f'ema_5_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'ema_7_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'ema_14_T-{i}'for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'ema_difference_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'supertrend_status_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'supertrend_crossover_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'supertrend_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "    #            [f'elastic_supertrend_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'elastic_supertrend_cross_T-{i}' for i in range(1, window_size + 1) ] + \\\n",
    "    #            [f'elastic_supertrend_status_T-{i}' for i in range(1, window_size + 1) ]\n",
    "\n",
    "    if temporal == False : \n",
    "        # [f'Day_of_Week_T-{i}' for i in range(1, window_size + 1)]  +\n",
    "        features =  [f'candle_type_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "               [f'rsi_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "               [f'Price_Range_T-{i}' for i in range(1, window_size + 1)  ] +\\\n",
    "               [f'rsi_crossover_slow_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "               [f'rsi_crossover_fast_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "               [f'harmonic_mean_low_T-{i}' for i in range(1, window_size + 1)  ] +\\\n",
    "               [f'harmonic_mean_high_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "               [f'slow_harmonic_mean_T-{i}' for i in range(1, window_size + 1) ] +\\\n",
    "               [f'fast_harmonic_mean_T-{i}' for i in range(1, window_size + 1) ] \n",
    "        \n",
    "    day_features_X   =  day_data[features]\n",
    "\n",
    "    day_features_X = day_features_X.apply(pd.to_numeric, errors='coerce')\n",
    "\n",
    "    day_features_X = day_features_X.astype(float)\n",
    "\n",
    "    hour4_features_X = hour4_data\n",
    "\n",
    "    hour4_features_X = hour4_features_X.apply(pd.to_numeric, errors='coerce')\n",
    "\n",
    "    hour4_features_X = hour4_features_X.astype(float)\n",
    "    \n",
    "    # substrings = ['relative_range', 'candle_type', 'heikin_ashi', 'stdev', 'true' ] #, 'heikin_ashi',  'supertrend_h4']\n",
    "\n",
    "    # substrings = ['relative_range', 'candle_type', 'supertrend_h4' ] #, 'heikin_ashi',  'supertrend_h4']\n",
    "    # substrings = [ 'supertrend_h4', 'candle_type', 'crossover', 'slow_harmonic', 'fast_harmonic' ]\n",
    "    # substrings = [ 'supertrend_h4' ]\n",
    "    \n",
    "    if alpha_type == \"gamma\":\n",
    "        substrings = [ 'supertrend_h4' ]\n",
    "\n",
    "    elif alpha_type == \"beta\":\n",
    "        substrings = [ 'supertrend_h4', 'candle_type']\n",
    "        \n",
    "        \n",
    "    elif alpha_type == \"omega\":\n",
    "        substrings = [ 'supertrend_h4', 'candle_type', 'rsi_slow_crossover']\n",
    "        \n",
    "    elif alpha_type == \"alpha\":\n",
    "        substrings = [ 'relative_range', 'fast_harmonic_mean']\n",
    "    \n",
    "    elif alpha_type == \"delta\":\n",
    "        substrings = [ 'relative_range', 'rsi_slow_crossover', 'candle_type', 'supertrend_h4']\n",
    "    \n",
    "    elif alpha_type == \"ha\":\n",
    "        substrings = [ 'stdev_slow', 'heikin_ashi', 'slow_harmonic_mean'  ] # 'slow_harmonic_mean'\n",
    "    \n",
    "    elif alpha_type == \"rsi\":\n",
    "        substrings = [ 'rsi_sma_slow', 'candle_type', 'slow_harmonic_mean', 'supertrend_h4' ] # 'slow_harmonic_mean' ]\n",
    "        # features += [f'High_T-{i}' for i in range(1, window_size + 1) ] +\n",
    "        # [f'High_T-{i}' for i in range(1, window_size + 1) ] +  [f'Low_T-{i}'  for i in range(1, window_size + 1) ]\n",
    "    elif alpha_type == \"lambda\":\n",
    "        substrings = ['relative_range', 'candle_type', 'heikin_ashi' ]\n",
    "\n",
    "    # substrings = [ 'supertrend_h4', 'candle_type' ]\n",
    "    # substrings = [ 'supertrend_h4' ] \n",
    "    # # Filter columns that contain any of the specified substrings\n",
    "    filtered_columns = [col for col in hour4_features_X.columns if any(sub in col for sub in substrings)]\n",
    "    hour4_features_X = hour4_features_X[filtered_columns]\n",
    "    # hour4_features_X = hour4_features_X.filter(like='relative_range')\n",
    "\n",
    "    X = day_features_X.join(hour4_features_X)\n",
    "    # X = day_features_X\n",
    "    \n",
    "    # if temporal == False :  X = day_features_X\n",
    "    \n",
    "    # X = day_features_X.join(hour4_features_X)\n",
    "\n",
    "    y = day_data['Target']\n",
    "\n",
    "    X.fillna(0.0, inplace=True)\n",
    "\n",
    "    y.fillna(0.0, inplace=True)\n",
    "\n",
    "    # print(len(data[features]))\n",
    "    X = X.astype(float)\n",
    "\n",
    "    div = int(len(X) * 0.8)\n",
    "    # train = :div\n",
    "    # test  = div:\n",
    "    X_train = X[:div]\n",
    "    X_test  = X[div:]\n",
    "\n",
    "    y_train = y[:div]\n",
    "    y_test  = y[div:]\n",
    "\n",
    "    # # Apply RobustScaler\n",
    "    scaler = RobustScaler()\n",
    "    \n",
    "    # scaler = StandardScaler()\n",
    "\n",
    "    X_train_scaled = scaler.fit_transform(X_train)\n",
    "    # X_test_scaled = scaler.transform(X_test)\n",
    "    X_test_scaled = scaler.fit_transform(X_test)\n",
    "    \n",
    "    if not X_test_re:\n",
    "        return X_train_scaled, X_test_scaled, y_train, y_test\n",
    "    else :\n",
    "        return X_train_scaled, X_test_scaled, y_train, y_test, X_test\n",
    "\n",
    "def train_model_2(X_train_scaled, X_test_scaled, y_train, y_test ):\n",
    "    \n",
    "    # # Convert to Pool (CatBoost-specific data structure)\n",
    "    train_pool = Pool(X_train_scaled, y_train)\n",
    "    test_pool = Pool(X_test_scaled, y_test)\n",
    "    \n",
    "    \n",
    "    np.random.seed(42)\n",
    "\n",
    "\n",
    "    # Initialize CatBoost regressor\n",
    "    model = CatBoostRegressor(iterations=300, early_stopping_rounds=75, \n",
    "                              learning_rate=0.1 , depth=5, verbose=100)\n",
    "\n",
    "    # Train the model using train pool\n",
    "    model.fit(train_pool)\n",
    "\n",
    "\n",
    "    # Make predictions using test pool\n",
    "    y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "    # Evaluate the model\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    \n",
    "    print(f'Mean Squared Error: {mse}')\n",
    "    \n",
    "    return model\n",
    "\n",
    "def finetune_model(X_train_scaled, X_test_scaled, y_train, y_test, parameters = None ):\n",
    "\n",
    "#     model = CatBoostRegressor()\n",
    "#     parameters = {'depth' : [6,8,10],'learning_rate' : [0.01, 0.05, 0.1],\n",
    "#                   'iterations'    : [30, 50, 100]}\n",
    "\n",
    "#     model = GridSearchCV(estimator=model, param_grid = parameters, cv = 2, n_jobs=-1)\n",
    "#     model.fit(X_train_scaled, y_train)\n",
    "    \n",
    "#     # Make predictions using test pool\n",
    "#     y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "#     # Evaluate the model\n",
    "#     mse = mean_squared_error(y_test, y_pred)\n",
    "    \n",
    "#     print(f'Mean Squared Error: {mse}')\n",
    "\n",
    "        # Create the model\n",
    "    rf_regressor = RandomForestRegressor(random_state=42)\n",
    "\n",
    "    # Define the parameter distribution\n",
    "    param_dist = {\n",
    "        'n_estimators': randint(50, 200),\n",
    "        'max_features': ['auto', 'sqrt', 'log2'],\n",
    "        'max_depth': [None, 10, 20, 30],\n",
    "        'min_samples_split': randint(2, 11),\n",
    "        'min_samples_leaf': randint(1, 5),\n",
    "        'bootstrap': [True, False]\n",
    "    }\n",
    "\n",
    "    # Create the RandomizedSearchCV object\n",
    "    model = RandomizedSearchCV(estimator=rf_regressor, param_distributions=param_dist,\n",
    "                                       n_iter=100, cv=3, n_jobs=-1, verbose=2, scoring='neg_mean_squared_error',\n",
    "                                       random_state=42)\n",
    "\n",
    "    # Fit the random search to the data\n",
    "    model.fit(X_train_scaled, y_train)\n",
    "\n",
    "    # Get the best parameters and best score\n",
    "    best_params = random_search.best_params_\n",
    "    best_score  = random_search.best_score_\n",
    "\n",
    "    # Make predictions using test pool\n",
    "    y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "    # Evaluate the model\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    \n",
    "    print(f'Mean Squared Error: {mse}')\n",
    "    \n",
    "    print(\"Best Parameters: \",best_params)\n",
    "    print(\"Best Score: \",best_score)\n",
    "    \n",
    "    return model\n",
    "    \n",
    "    \n",
    "def train_model(X_train_scaled, X_test_scaled, y_train, y_test, parameters = None ):\n",
    "    \n",
    "    # train_model_1\n",
    "    # # Convert to Pool (CatBoost-specific data structure)\n",
    "    train_pool = Pool(X_train_scaled, y_train)\n",
    "    test_pool = Pool(X_test_scaled, y_test)\n",
    "    \n",
    "    \n",
    "    # np.random.seed(42)\n",
    "\n",
    "\n",
    "    # Initialize CatBoost regressor\n",
    "    if not parameters:\n",
    "        model = CatBoostRegressor(iterations=300, early_stopping_rounds=75, \n",
    "                              learning_rate=0.01 , depth=6, verbose=100)\n",
    "    \n",
    "    else : \n",
    "        iteration, learning_rate, depth = parameters[0], parameters[1], parameters[2]\n",
    "        model = CatBoostRegressor(iterations=iteration, early_stopping_rounds=75, \n",
    "                              learning_rate=learning_rate, depth=depth, verbose=100)\n",
    "        \n",
    "    # Train the model using train pool\n",
    "    model.fit(train_pool)\n",
    "\n",
    "\n",
    "    # Make predictions using test pool\n",
    "    y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "    # Evaluate the model\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    \n",
    "    print(f'Mean Squared Error: {mse}')\n",
    "    \n",
    "    return model\n",
    "\n",
    "    \n",
    "    \n",
    "def train_model_1(X_train_scaled, X_test_scaled, y_train, y_test ):\n",
    "    \n",
    "    # train_model_1\n",
    "    # # Convert to Pool (CatBoost-specific data structure)\n",
    "    train_pool = Pool(X_train_scaled, y_train)\n",
    "    test_pool = Pool(X_test_scaled, y_test)\n",
    "    \n",
    "    \n",
    "    np.random.seed(42)\n",
    "\n",
    "\n",
    "    # Initialize CatBoost regressor\n",
    "    model = CatBoostRegressor(iterations=300, early_stopping_rounds=75, \n",
    "                              learning_rate=0.01 , depth=6, verbose=100)\n",
    "\n",
    "    # Train the model using train pool\n",
    "    model.fit(train_pool)\n",
    "\n",
    "\n",
    "    # Make predictions using test pool\n",
    "    y_pred = model.predict(X_test_scaled)\n",
    "\n",
    "    # Evaluate the model\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    \n",
    "    print(f'Mean Squared Error: {mse}')\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "# def load_features_files(hour4_lag_3=None):\n",
    "\n",
    "#     # day_features_path = 'day_features_data_lagby_14_v2.bin'\n",
    "\n",
    "#     with open(day_features_path, 'rb') as file :\n",
    "\n",
    "#         day_features = pickle.load(file)\n",
    "\n",
    "\n",
    "#     # hour4_features_path = 'hour4_features_data.bin'  \n",
    "#     # hour4_features_path = 'hour4_features_data_lag_by_7.bin'\n",
    "#     hour4_features_path = 'hour4_features_data_lag_by_3_v2.bin'\n",
    "    \n",
    "#     if hour4_lag_3 == True : hour4_features_path = 'hour4_features_data_lag_by_3_v2.bin'\n",
    "     \n",
    "#     with open(hour4_features_path, 'rb') as file :\n",
    "\n",
    "#         hour4_features = pickle.load(file)\n",
    "    \n",
    "#     return  day_features, hour4_features\n",
    "    \n",
    "\n",
    "def evaluate_model_old(symbol, model, X_test_scaled,  y_test):\n",
    "    \n",
    "    sample = 0\n",
    "    right  = 0\n",
    "    wrong  = 0\n",
    "    status = []\n",
    "    pips_profit = 0 \n",
    "    pips_loss = 0\n",
    "\n",
    "    for point in range(1, 22 + 1 ):\n",
    "\n",
    "        sample += 1\n",
    "        y_pred = model.predict(X_test_scaled[-point])\n",
    "        y_true = y_test[-point]\n",
    "\n",
    "        if y_pred < 0.0 and y_true < 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        elif y_pred > 0.0 and y_true > 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        else:\n",
    "            pips_loss += abs(y_true)\n",
    "            wrong += 1\n",
    "            status.append(0)\n",
    "\n",
    "    # y_pred = model.predict(test_pool )\n",
    "\n",
    "        # print(\"actual : \",y_test[-3],\"\\npredicted : \",y_pred )\n",
    "    print(f\"{symbol}\\n\")\n",
    "    print(\"accuracy on last 22 test points Month May 2024 with low learning rate 2020 dataset 80% training dataset is \",right/sample * 100 )\n",
    "\n",
    "    # print(\"accuracy on last 100 test points is \",right) #wrong/sample * 100 )\n",
    "    print(status)\n",
    "    print(f\"Pips On Profit Side was : \",pips_profit)\n",
    "    print(\"Pips On Loss Side was : \",pips_loss)\n",
    "\n",
    "    print(\"net pips profit : \", pips_profit-pips_loss )\n",
    "    \n",
    "    \n",
    "    status = []\n",
    "    for point in range(1, len(X_test_scaled) + 1 ):\n",
    "\n",
    "        sample += 1\n",
    "        y_pred = model.predict(X_test_scaled[-point])\n",
    "        y_true = y_test[-point]\n",
    "\n",
    "        if y_pred < 0.0 and y_true < 0.0 :\n",
    "            # pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        elif y_pred > 0.0 and y_true > 0.0 :\n",
    "            # pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        else:\n",
    "            # pips_loss += abs(y_true)\n",
    "            wrong += 1\n",
    "            status.append(0)\n",
    "\n",
    "    print(\"accuracy on all test points with low learning rate 2020 dataset 80% training dataset is \",right/sample * 100 )\n",
    "\n",
    "    # print(\"accuracy on last 100 test points is \",right) #wrong/sample * 100 )\n",
    "    print(status)\n",
    "    \n",
    "    \n",
    "    print(f\"{symbol}\\n==============================\")\n",
    "    \n",
    "\n",
    "def save_model_v2(symbol, model, parameters, alpha_type, window_size):\n",
    "    \n",
    "    import pickle\n",
    "    \n",
    "    # parameters = [ iteration, learning_rate, depth ]\n",
    "    iterations = parameters[0]\n",
    "    learning_rate = parameters[1] \n",
    "    depth = parameters[2]\n",
    "    model_file = {\"model\": model ,  \"model_info\": { \"learning_rate\": learning_rate, \"depth\": depth, \n",
    "                  \"iterations\": iterations, \"alpha_type\": alpha_type, \"window_size\": window_size,\n",
    "                  \"scaling\": \"robust_scaling\", \"version\": \"v2\" }, \"symbol\": symbol+\"_pips_change\" }\n",
    "    with open(f\"v2_forex_models/{symbol}_model_pips_change\", 'wb') as file:\n",
    "        \n",
    "        pickle.dump(model_file, file)\n",
    "    print(\"saved successfully\")\n",
    "        \n",
    "    # model.save_model(f\"v2_forex_models/{symbol}_model_pips_change\")\n",
    "\n",
    "    \n",
    "    \n",
    "def save_model(symbol, model):\n",
    "    \n",
    "    model.save_model(f\"v2_forex_models/{symbol}_model_pips_change\")\n",
    "\n",
    "    \n",
    "    \n",
    "def load_features_files():\n",
    "\n",
    "    day_features_path = 'day_features_data_lagby_14_v2.bin' \n",
    "    # day_features_path = 'day_features_data_lagby_14_v3.bin'\n",
    "\n",
    "\n",
    "    with open(day_features_path, 'rb') as file :\n",
    "\n",
    "        day_features = pickle.load(file)\n",
    "\n",
    "\n",
    "    # hour4_features_path = 'hour4_features_data.bin'  \n",
    "    hour4_features_path = 'hour4_features_data_lag_by_3_v2.bin'\n",
    "\n",
    "    with open(hour4_features_path, 'rb') as file :\n",
    "\n",
    "        hour4_features = pickle.load(file)\n",
    "        \n",
    "    return  day_features, hour4_features\n",
    "    \n",
    "    \n",
    "def get_day_hour4_features(symbol, day_features, hour4_features):\n",
    "    \n",
    "    \n",
    "    \n",
    "    for day_pair in day_features: \n",
    "        if  day_pair['symbol'] == symbol: \n",
    "\n",
    "            day_data = day_pair['day_features'] \n",
    "            break\n",
    "\n",
    "\n",
    "    for hour4_pair in hour4_features: \n",
    "        if  hour4_pair['symbol'] == symbol: \n",
    "\n",
    "            hour4_data = hour4_pair['hour4_features'] \n",
    "            break\n",
    "    \n",
    "    return  day_data, hour4_data\n",
    "\n",
    "def load_model(symbol, path = None):\n",
    "    \n",
    "    \n",
    "    if path: \n",
    "        pass\n",
    "        path = f\"{path}/{symbol}_model_pips_change\"\n",
    "        \n",
    "    else: \n",
    "        path = f\"v2_forex_models/{symbol}_model_pips_change\"\n",
    "    \n",
    "    model = CatBoostRegressor()\n",
    "    \n",
    "    model.load_model(path)\n",
    "    return model\n",
    "\n",
    "\n",
    "# def load_model_old(symbol):\n",
    "    \n",
    "#     pass\n",
    "#     model = CatBoostRegressor()\n",
    "#     path = f'forex_models_low_lr_few_h4features_300 iteration/{symbol}_model_pips_change'\n",
    "#     model.load_model(path)\n",
    "#     return model\n",
    "\n",
    "\n",
    "def evaluate_model(symbol, model, X_test_scaled, y_test):\n",
    "    \n",
    "    sample = 0\n",
    "    right  = 0\n",
    "    wrong  = 0\n",
    "    status = []\n",
    "    pips_profit = 0 \n",
    "    pips_loss = 0\n",
    "\n",
    "    for point in range(23, 45 + 1 ): # may 1 - 22 # april 23 - 55\n",
    "\n",
    "        sample += 1\n",
    "        y_pred = model.predict(X_test_scaled[-point])\n",
    "        y_true = y_test[-point]\n",
    "\n",
    "        if y_pred < 0.0 and y_true < 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        elif y_pred > 0.0 and y_true > 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        else:\n",
    "            pips_loss += abs(y_true)\n",
    "            wrong += 1\n",
    "            status.append(0)\n",
    "\n",
    "    # y_pred = model.predict(test_pool )\n",
    "\n",
    "        # print(\"actual : \",y_test[-3],\"\\npredicted : \",y_pred )\n",
    "    print(f\"{symbol}\\n\")\n",
    "    print(\"accuracy on last 22 test points Month May 2024 with low learning rate 2020 dataset 80% training dataset is \",right/sample * 100 )\n",
    "\n",
    "    # print(\"accuracy on last 100 test points is \",right) #wrong/sample * 100 )\n",
    "    print(status)\n",
    "    print(f\"Pips On Profit Side was : \",pips_profit)\n",
    "    print(\"Pips On Loss Side was : \",pips_loss)\n",
    "\n",
    "    print(\"net pips profit : \", pips_profit-pips_loss )\n",
    "    \n",
    "    status = []\n",
    "    for point in range(1, len(X_test_scaled) + 1 ):\n",
    "\n",
    "        sample += 1\n",
    "        y_pred = model.predict(X_test_scaled[-point])\n",
    "        y_true = y_test[-point]\n",
    "\n",
    "        if y_pred < 0.0 and y_true < 0.0 :\n",
    "            # pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        elif y_pred > 0.0 and y_true > 0.0 :\n",
    "            # pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        else:\n",
    "            # pips_loss += abs(y_true)\n",
    "            wrong += 1\n",
    "            status.append(0)\n",
    "\n",
    "    print(\"accuracy on all test points excluding last 22 points with low learning rate 2020 dataset 80% training dataset is \",right/sample * 100 )\n",
    "\n",
    "    # print(\"accuracy on last 100 test points is \",right) #wrong/sample * 100 )\n",
    "    print(status)\n",
    "    \n",
    "    \n",
    "    print(f\"{symbol}\\n==============================\")\n",
    "    # try:\n",
    "    #     return  pips_profit-pips_loss, right/sample * 100\n",
    "    # except : pass\n",
    "\n",
    "    # net_gains , accuracy\n",
    "    \n",
    "def evaluate_model_return_gains_accuracy(symbol, model, X_test_scaled, y_test, custom_sample= None, X_test = None ):\n",
    "    \n",
    "    sample = 0\n",
    "    right  = 0\n",
    "    wrong  = 0\n",
    "    status = []\n",
    "    pips_profit = 0\n",
    "    pips_loss = 0\n",
    "    \n",
    "    # custom_sample=[1,22]\n",
    "    if custom_sample:\n",
    "        step, cluster = custom_sample[0] , custom_sample[1] \n",
    "        start , end = 0 , 0\n",
    "        # for i in range(step):\n",
    "        #     start += 1 + end # 23\n",
    "        #     end   += cluster      # \n",
    "        \n",
    "        for i in range(step): # 1-> 22 + 1 # 2\n",
    "            end   += cluster\n",
    "        end += 1\n",
    "        start = end - cluster\n",
    "        \n",
    "    else : start, end = 1, 22\n",
    "    \n",
    "#     # Convert net gains to numpy array\n",
    "# returns = np.array(net_gains)\n",
    "\n",
    "# # Calculate mean return\n",
    "# mean_return = np.mean(returns)\n",
    "\n",
    "# # Calculate standard deviation of returns\n",
    "# std_return = np.std(returns)\n",
    "\n",
    "# # Assuming risk-free rate is 0 for simplicity\n",
    "# risk_free_rate = 0\n",
    "\n",
    "# # Calculate Sharpe ratio\n",
    "# sharpe_ratio = (mean_return - risk_free_rate) / std_return\n",
    "\n",
    "# print(f'Sharpe Ratio: {sharpe_ratio:.2f}')\n",
    "\n",
    "\n",
    "    net_gains_list = []\n",
    "    for point in range(start, end  ): # may 1 - 22 # april 23 - 45\n",
    "\n",
    "        sample += 1\n",
    "        y_pred = model.predict(X_test_scaled[-point])\n",
    "        y_true = y_test[-point]\n",
    "\n",
    "        if y_pred < 0.0 and y_true < 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            net_gains_list.append(abs(y_true))\n",
    "            status.append(-1)\n",
    "\n",
    "        elif y_pred > 0.0 and y_true > 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            net_gains_list.append(y_true)\n",
    "            status.append(1)\n",
    "\n",
    "        else:\n",
    "            pips_loss += abs(y_true)\n",
    "            wrong += 1\n",
    "            # status.append(0)\n",
    "            net_gains_list.append(-1*abs(y_true))\n",
    "\n",
    "            if y_pred > 0.0 and y_true < 0.0 : status.append(str('+1-1'))\n",
    "\n",
    "            elif y_pred < 0.0 and y_true > 0.0 : status.append(str('-1+1'))\n",
    "        \n",
    "\n",
    "    # y_pred = model.predict(test_pool )\n",
    "    \n",
    "    # print(\"actual : \",y_test[-3],\"\\npredicted : \",y_pred )\n",
    "    # print(f\"{symbol}\\n\")\n",
    "    # print(\"accuracy on last 22 test points Month May 2024 with low learning rate 2020 dataset 80% training dataset is \",right/sample * 100 )\n",
    "    \n",
    "    returns = np.array(net_gains_list)\n",
    "\n",
    "    # Calculate mean return\n",
    "    mean_return = np.mean(returns)\n",
    "\n",
    "    # Calculate standard deviation of returns\n",
    "    std_return = np.std(returns)\n",
    "\n",
    "    # Assuming risk-free rate is 0 for simplicity\n",
    "    risk_free_rate = 0\n",
    "\n",
    "    # Calculate Sharpe ratio\n",
    "    sharpe_ratio = (mean_return - risk_free_rate) / std_return\n",
    "    if len(X_test) != None :\n",
    "        date   = X_test.index.values[-end]\n",
    "        date = str(date.astype('datetime64[D]'))\n",
    "        print(symbol , f\" Starting date : {date} \")\n",
    "        \n",
    "    status = status[::-1] # starting from end because end is the first most day of the chosen month\n",
    "    \n",
    "    print(status)\n",
    "    # print(\"accuracy on last 100 test points is \",right) #wrong/sample * 100 )\n",
    "    accuracy_by_days = right/sample * 100\n",
    "    # print(f\"Accuracy by days :\",accuracy_by_days)\n",
    "    print(f'Sharpe Ratio: {sharpe_ratio:.2f}')\n",
    "    \n",
    "    print(f\"Pips On Profit Side was : \",pips_profit)\n",
    "    print(\"Pips On Loss Side was : \",pips_loss)\n",
    "    accuracy_by_net_gains = pips_profit / (pips_profit+pips_loss)*100\n",
    "    print('accuracy by days', accuracy_by_days )\n",
    "    \n",
    "    print('accuracy_by_net_gains ',accuracy_by_net_gains )\n",
    "    net_gains = pips_profit-pips_loss\n",
    "#     print(\"net pips profit : \", pips_profit-pips_loss )\n",
    "    \n",
    "    status = []\n",
    "    right, wrong = 0, 0\n",
    "    sample = 0\n",
    "    pips_profit, pips_loss = 0, 0\n",
    "    for point in range(end, len(X_test_scaled) + 1 ): # excluding May April 55  # excluding May 23 -> till end of length\n",
    "\n",
    "        sample += 1\n",
    "        y_pred = model.predict(X_test_scaled[-point])\n",
    "        y_true = y_test[-point]\n",
    "\n",
    "        if y_pred < 0.0 and y_true < 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(-1)\n",
    "\n",
    "        elif y_pred > 0.0 and y_true > 0.0 :\n",
    "            pips_profit += abs(y_true)\n",
    "            right += 1\n",
    "            status.append(1)\n",
    "\n",
    "        else:\n",
    "            pips_loss += abs(y_true)\n",
    "            wrong += 1\n",
    "            # status.append(0)\n",
    "            if y_pred > 0.0 and y_true < 0.0 : status.append(str('+1-1'))\n",
    "\n",
    "            elif y_pred < 0.0 and y_true > 0.0 : status.append(str('-1+1'))\n",
    "\n",
    "    # print(\"accuracy on all test points excluding last 22 points with low learning rate 2020 dataset 80% training dataset is \",right/sample * 100 )\n",
    "\n",
    "    # print(\"accuracy on last 100 test points is \",right) #wrong/sample * 100 )\n",
    "    # print(status)\n",
    "    accuracy_by_days_before = right/sample * 100\n",
    "    accuracy_by_net_gains_before = pips_profit / (pips_profit+pips_loss)*100\n",
    "    gains = {}\n",
    "    gains['net_gains'] = net_gains\n",
    "    gains['accuracy_by_days']  = accuracy_by_days\n",
    "    gains['accuracy_by_days_before']  = accuracy_by_days_before\n",
    "    gains['accuracy_by_net_gains_before'] = accuracy_by_net_gains_before\n",
    "    gains['accuracy_by_net_gains'] = accuracy_by_net_gains\n",
    "    gains['sharpe_ratio'] = sharpe_ratio\n",
    "    \n",
    "    # print(f\"{symbol}\\n==============================\")\n",
    "    try:\n",
    "        return  gains, accuracy_by_days_before\n",
    "    except : pass\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def update_profit_and_loss(pl_info, date, pips_profit, pips_loss, net_gain):\n",
    "    if date in pl_info:\n",
    "        pl_info[date]['pips_profit'] += pips_profit\n",
    "        pl_info[date]['pips_loss'] += pips_loss\n",
    "        pl_info[date]['net_gain'] += net_gain\n",
    "    else:\n",
    "        pl_info[date] = {'pips_profit': pips_profit, 'pips_loss': pips_loss, 'net_gain': net_gain}\n",
    "\n",
    "    return pl_info\n",
    "        \n",
    "\n",
    "def each_day_gain_loss(day_features, hour4_features, custom_sample= None , model_path = None):\n",
    "\n",
    "    pass\n",
    "    # custom_sample = [ 1, 22 ]  \n",
    "    forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY']\n",
    "    symbols_store = {}\n",
    "    \n",
    "    sample = 0\n",
    "    right  = 0\n",
    "    wrong  = 0\n",
    "    status = []\n",
    "    pips_profit = 0\n",
    "    pips_loss = 0\n",
    "    pl_info = {}\n",
    "    # custom_sample=[1,22]\n",
    "    if custom_sample :\n",
    "        step, cluster = custom_sample[0] , custom_sample[1] \n",
    "        start , end = 0 , 0\n",
    "        for i in range(step):\n",
    "            end   += cluster + 1\n",
    "\n",
    "        start = end - cluster\n",
    "    \n",
    "    \n",
    "    else : start, end = 1, 22\n",
    "    \n",
    "    \n",
    "    for point in range(start, end + 1 ): # may 1 - 22 # april 23 - 45\n",
    "        \n",
    "        pips_profit, pips_loss = 0, 0\n",
    "        net_gains = 0\n",
    "        \n",
    "        sample += 1\n",
    "        for symbol in forex_pairs:\n",
    "    \n",
    "            \n",
    "            if symbol not in symbols_store:\n",
    "            \n",
    "                day_data, hour4_data = get_day_hour4_features(symbol, day_features, hour4_features)\n",
    "                X_train_scaled, X_test_scaled, y_train, y_test, X_test = get_features_transformed(symbol, day_data, hour4_data, date_resume='2020-06-01',  X_test_re=True)\n",
    "                \n",
    "                if model_path: model = load_model(symbol, path = model_path)\n",
    "                else : model = load_model(symbol)\n",
    "                \n",
    "                \n",
    "                symbols_store[symbol] = { 'X_test_scaled': X_test_scaled,  \n",
    "                                         'y_test': y_test, 'model': model, 'X_test': X_test  }\n",
    "                # symbols_store[symbol] = { 'y_test': y_test }\n",
    "                # symbols_store[symbol] = { 'model': model }\n",
    "                # symbols_store[symbol] = { 'X_test': X_test }\n",
    "            \n",
    "            else:\n",
    "                X_test_scaled = symbols_store[symbol]['X_test_scaled']\n",
    "                y_test = symbols_store[symbol]['y_test']\n",
    "                model  = symbols_store[symbol]['model']\n",
    "                X_test = symbols_store[symbol]['X_test']\n",
    "            \n",
    "            y_pred = model.predict(X_test_scaled[-point])\n",
    "            y_true = y_test[-point]\n",
    "            date   = X_test.index.values[-point]\n",
    "            # date = np.datetime64(date)\n",
    "            date = str(date.astype('datetime64[D]'))\n",
    "            \n",
    "            if y_pred < 0.0 and y_true < 0.0 :\n",
    "                pips_profit += abs(y_true)\n",
    "                right += 1\n",
    "                status.append(-1)\n",
    "\n",
    "            elif y_pred > 0.0 and y_true > 0.0 :\n",
    "                pips_profit += abs(y_true)\n",
    "                right += 1\n",
    "                status.append(1)\n",
    "\n",
    "            else:\n",
    "                pips_loss += abs(y_true)\n",
    "                wrong += 1\n",
    "                # status.append(0)\n",
    "                if y_pred > 0.0 and y_true < 0.0 : status.append(str('+1-1'))\n",
    "\n",
    "                elif y_pred < 0.0 and y_true > 0.0 : status.append(str('-1+1'))\n",
    "            \n",
    "        net_gain = int(pips_profit - pips_loss)\n",
    "        pl_info  =  update_profit_and_loss(pl_info, date, int(pips_profit), int(pips_loss), net_gain)\n",
    "\n",
    "\n",
    "    return pl_info\n",
    "\n",
    "\n",
    "def update_symbol_profit_and_loss(pl_info, date, pips_profit, pips_loss, net_gain, symbol):\n",
    "      \n",
    "#     symbol\n",
    "    # if 'date_info' not in pl_info:  pl_info['date_info'] = date\n",
    "#     if symbol in pl_info:\n",
    "#         pl_info[symbol]['pips_profit'] += pips_profit\n",
    "#         pl_info[symbol]['pips_loss'] += pips_loss\n",
    "#         pl_info[symbol]['net_gain'] += net_gain\n",
    "        \n",
    "#     else:\n",
    "#         # pl_info[date] = {'pips_profit': pips_profit, 'pips_loss': pips_loss, 'net_gain': net_gain}\n",
    "    pl_info[symbol] = {'pips_profit': pips_profit, 'pips_loss': pips_loss, 'net_gain': net_gain, 'date': date}\n",
    "\n",
    "        \n",
    "    return pl_info\n",
    "\n",
    "# '2024-01-29': {'pips_profit': 137, 'pips_loss': 1127, 'net_gain': -989} \n",
    "\n",
    "\n",
    "\n",
    "def each_symbol_gain_loss_by_date(day_features, hour4_features, custom_sample= None , model_path = None, date_val='2024-01-29'):\n",
    "\n",
    "    pass\n",
    "    # custom_sample = [ 1, 22 ]  \n",
    "    forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY']\n",
    "    symbols_store = {}\n",
    "    \n",
    "    sample = 0\n",
    "    right  = 0\n",
    "    wrong  = 0\n",
    "    status = []\n",
    "    pips_profit = 0\n",
    "    pips_loss = 0\n",
    "    pl_info = {}\n",
    "    # custom_sample=[1,22]\n",
    "    if custom_sample :\n",
    "        step, cluster = custom_sample[0] , custom_sample[1] \n",
    "        start , end = 0 , 0\n",
    "        for i in range(step):\n",
    "            end   += cluster + 1\n",
    "\n",
    "        start = end - cluster\n",
    "    \n",
    "    \n",
    "    else : start, end = 1, 22\n",
    "    \n",
    "    \n",
    "    for point in range(start, end + 1 ): # may 1 - 22 # april 23 - 45\n",
    "        \n",
    "        pips_profit, pips_loss = 0, 0\n",
    "        net_gains = 0\n",
    "        \n",
    "        sample += 1\n",
    "        for symbol in forex_pairs:\n",
    "            \n",
    "            pips_profit, pips_loss = 0, 0\n",
    "            if symbol not in symbols_store:\n",
    "            \n",
    "                day_data, hour4_data = get_day_hour4_features(symbol, day_features, hour4_features)\n",
    "                X_train_scaled, X_test_scaled, y_train, y_test, X_test = get_features_transformed(symbol, day_data, hour4_data, date_resume='2020-06-01',  X_test_re=True)\n",
    "                \n",
    "                if model_path: model = load_model(symbol, path = model_path)\n",
    "                else : model = load_model(symbol)\n",
    "                \n",
    "                \n",
    "                symbols_store[symbol] = { 'X_test_scaled': X_test_scaled,  \n",
    "                                         'y_test': y_test, 'model': model, 'X_test': X_test  }\n",
    "                # symbols_store[symbol] = { 'y_test': y_test }\n",
    "                # symbols_store[symbol] = { 'model': model }\n",
    "                # symbols_store[symbol] = { 'X_test': X_test }\n",
    "            \n",
    "            else:\n",
    "                X_test_scaled = symbols_store[symbol]['X_test_scaled']\n",
    "                y_test = symbols_store[symbol]['y_test']\n",
    "                model  = symbols_store[symbol]['model']\n",
    "                X_test = symbols_store[symbol]['X_test']\n",
    "            \n",
    "            \n",
    "            date   = X_test.index.values[-point]\n",
    "            # date = np.datetime64(date)\n",
    "            date = str(date.astype('datetime64[D]'))\n",
    "            \n",
    "            if date != date_val : break\n",
    "            \n",
    "            y_pred = model.predict(X_test_scaled[-point])\n",
    "            y_true = y_test[-point]\n",
    "            \n",
    "            if y_pred < 0.0 and y_true < 0.0 :\n",
    "                pips_profit = abs(y_true)\n",
    "                right += 1\n",
    "                status.append(-1)\n",
    "\n",
    "            elif y_pred > 0.0 and y_true > 0.0 :\n",
    "                pips_profit = abs(y_true)\n",
    "                right += 1\n",
    "                status.append(1)\n",
    "\n",
    "            else:\n",
    "                pips_loss = abs(y_true)\n",
    "                wrong += 1\n",
    "                # status.append(0)\n",
    "                if y_pred > 0.0 and y_true < 0.0 : status.append(str('+1-1'))\n",
    "\n",
    "                elif y_pred < 0.0 and y_true > 0.0 : status.append(str('-1+1'))\n",
    "            \n",
    "            net_gain = int(pips_profit - pips_loss) # net gain on symbol basis for a particular day\n",
    "            pl_info  =  update_symbol_profit_and_loss(pl_info, date, int(pips_profit), int(pips_loss), net_gain, symbol)\n",
    "\n",
    "\n",
    "    return pl_info\n",
    "\n",
    "\n",
    "\n",
    "def each_day_gain_loss_with_threshold(day_features, hour4_features, custom_sample= None , model_path = None, threshold = 5 ):\n",
    "\n",
    "    pass\n",
    "    # custom_sample = [ 1, 22 ]  \n",
    "    forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY']\n",
    "    symbols_store = {}\n",
    "    \n",
    "    sample = 0\n",
    "    right  = 0\n",
    "    wrong  = 0\n",
    "    status = []\n",
    "    pips_profit = 0\n",
    "    pips_loss = 0\n",
    "    pl_info = {}\n",
    "    # custom_sample=[1,22]\n",
    "    if custom_sample :\n",
    "        step, cluster = custom_sample[0] , custom_sample[1] \n",
    "        start , end = 0 , 0\n",
    "        for i in range(step):\n",
    "            end   += cluster + 1\n",
    "\n",
    "        start = end - cluster\n",
    "    \n",
    "    \n",
    "    else : start, end = 1, 22\n",
    "    \n",
    "    era = start + 10\n",
    "    \n",
    "    for point in range(start, end + 1 ): # may 1 - 22 # april 23 - 45\n",
    "        \n",
    "        pips_profit, pips_loss = 0, 0\n",
    "        net_gains = 0\n",
    "        \n",
    "        sample += 1\n",
    "        for symbol in forex_pairs:\n",
    "    \n",
    "            \n",
    "            if symbol not in symbols_store:\n",
    "            \n",
    "                day_data, hour4_data = get_day_hour4_features(symbol, day_features, hour4_features)\n",
    "                X_train_scaled, X_test_scaled, y_train, y_test, X_test = get_features_transformed(symbol, day_data, hour4_data, date_resume='2020-06-01',  X_test_re=True)\n",
    "                \n",
    "                if model_path: model = load_model(symbol, path = model_path)\n",
    "                else : model = load_model(symbol)\n",
    "                \n",
    "                \n",
    "                symbols_store[symbol] = { 'X_test_scaled': X_test_scaled,  \n",
    "                                         'y_test': y_test, 'model': model, 'X_test': X_test  }\n",
    "                # symbols_store[symbol] = { 'y_test': y_test }\n",
    "                # symbols_store[symbol] = { 'model': model }\n",
    "                # symbols_store[symbol] = { 'X_test': X_test }\n",
    "            \n",
    "            else:\n",
    "                X_test_scaled = symbols_store[symbol]['X_test_scaled']\n",
    "                y_test = symbols_store[symbol]['y_test']\n",
    "                model  = symbols_store[symbol]['model']\n",
    "                X_test = symbols_store[symbol]['X_test']\n",
    "            \n",
    "            y_pred = model.predict(X_test_scaled[-point])\n",
    "            y_true = y_test[-point]\n",
    "            date   = X_test.index.values[-point]\n",
    "            # date = np.datetime64(date)\n",
    "            date = str(date.astype('datetime64[D]'))\n",
    "            if point < era : \n",
    "                if abs(y_pred) < threshold : continue\n",
    "            \n",
    "            if y_pred < 0.0 and y_true < 0.0 :\n",
    "                pips_profit += abs(y_true)\n",
    "                right += 1\n",
    "                status.append(-1)\n",
    "\n",
    "            elif y_pred > 0.0 and y_true > 0.0 :\n",
    "                pips_profit += abs(y_true)\n",
    "                right += 1\n",
    "                status.append(1)\n",
    "\n",
    "            else:\n",
    "                pips_loss += abs(y_true)\n",
    "                wrong += 1\n",
    "                # status.append(0)\n",
    "                if y_pred > 0.0 and y_true < 0.0 : status.append(str('+1-1'))\n",
    "\n",
    "                elif y_pred < 0.0 and y_true > 0.0 : status.append(str('-1+1'))\n",
    "            \n",
    "        net_gain = int(pips_profit - pips_loss)\n",
    "        pl_info  =  update_profit_and_loss(pl_info, date, int(pips_profit), int(pips_loss), net_gain)\n",
    "\n",
    "\n",
    "    return pl_info\n",
    "\n",
    "#     accuracy_by_days = right/sample * 100\n",
    "\n",
    "#     accuracy_by_net_gains = pips_profit / (pips_profit+pips_loss)*100\n",
    "\n",
    "#     net_gains = pips_profit-pips_loss\n",
    "\n",
    "    \n",
    "import pickle\n",
    "\n",
    "parameters = {\n",
    "    'NZDJPY': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.001,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 6\n",
    "    },\n",
    "    'AUDJPY': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.01,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 6\n",
    "    },\n",
    "    'USDJPY': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.01,\n",
    "        'depth': 6\n",
    "    },\n",
    "    'CADJPY': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.01,\n",
    "        'depth': 6\n",
    "    },\n",
    "    'NZDCAD': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 114\n",
    "    },\n",
    "    'CADCHF': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 158\n",
    "    },\n",
    "    'USDCHF2': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 50,\n",
    "        'net pips profit': 179\n",
    "    },\n",
    "    'EURAUD': {\n",
    "        'Iterations': 7,\n",
    "        'Lr': 0.06,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 56,\n",
    "        'net pips profit': 306\n",
    "    },\n",
    "    'EURNZD3': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 278\n",
    "    },\n",
    "    'GBPNZD': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 50,\n",
    "        'net pips profit': 224\n",
    "    },\n",
    "    'NZDCHF2': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 50,\n",
    "        'net pips profit': 140\n",
    "    },\n",
    "    'EURUSD3': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 56,\n",
    "        'net pips profit': 306\n",
    "    },\n",
    "    'GBPCAD': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.001,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 54,\n",
    "        'net pips profit': 151\n",
    "    },\n",
    "    'GBPUSD2': {\n",
    "        'Iterations': 5,\n",
    "        'Lr': 0.001,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 52,\n",
    "        'net pips profit': 217\n",
    "    },\n",
    "    'EURGBP': {\n",
    "        'Lr': 0.001 ,\n",
    "        'Iterations': 188,\n",
    "        'depth': 7,\n",
    "        'accuracy' :52,\n",
    "        'pips net profit' : 97\n",
    "    },\n",
    "    'EURCAD2': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 7,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 51,\n",
    "        'pips net profit':  228\n",
    "    }, \n",
    "    'GBPCHF': {\n",
    "        'Iterations': 15,\n",
    "        'Lr': 0.1,\n",
    "        'scaler': 'RobustScaler()',\n",
    "        'depth': 8,\n",
    "        'window_size': 9,\n",
    "        'accuracy': 53,\n",
    "        'pips net profit':  241\n",
    "    }, \n",
    "    'NZDUSD': {\n",
    "        'Iterations': 5, \n",
    "        'Lr': 0.77, \n",
    "        'depth': 7,\n",
    "        'accuracy': 54\n",
    "    }, \n",
    "    'EURUSD2': {\n",
    "        'Lr':0.025,\n",
    "        'Iterations': 450, \n",
    "        'depth': 5,\n",
    "        'accuracy': 53\n",
    "    },\n",
    "    'EURNZD2':{\n",
    "     'Iterations': 29, \n",
    "     'Lr': 0.63, \n",
    "     'depth': 7,\n",
    "     'accuracy': 50\n",
    "    }, \n",
    "    'GBPUSD': {\n",
    "     'Iterations': 13, \n",
    "     'Lr': 0.77, \n",
    "     'depth': 7,\n",
    "     'accuracy': 53\n",
    "    },\n",
    "    'GBPAUD': {\n",
    "    'Iterations': 13,\n",
    "    'Lr': 0.77,\n",
    "    'depth': 8,\n",
    "    'accuracy': 55       \n",
    "    },\n",
    "    'AUDUSD2': {\n",
    "    'Iterations': 5,\n",
    "    'Lr': 0.7,\n",
    "    'depth': 7,\n",
    "    'accuracy': 50       \n",
    "    },\n",
    "    'AUDCHF': {\n",
    "    'Iterations': 7,\n",
    "    'Lr': 0.71,\n",
    "    'depth' : 7\n",
    "    }, \n",
    "    'AUDCAD':{\n",
    "    'Iterations' : 9,\n",
    "    'Lr': 0.67,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'AUDNZD':{\n",
    "    'Iterations': 15,\n",
    "    'Lr': 0.68,\n",
    "    'depth': 7 \n",
    "    }, \n",
    "    'EURCAD':{ \n",
    "    'Iterations': 20,\n",
    "    'Lr': 0.039,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'NZDCHF' : {\n",
    "    'Iterations': 27,\n",
    "    'Lr' : 0.35,\n",
    "    'depth' : 7\n",
    "    },\n",
    "    'USDCAD': {\n",
    "    'Iterations': 20,\n",
    "    'Lr': 0.039,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'USDCHF':{\n",
    "    'Iterations': 188,\n",
    "    'Lr': 0.67,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'NZDCAD':{\n",
    "    'Iterations': 17,\n",
    "    'Lr': 0.38,\n",
    "    'depth': 7\n",
    "    }, \n",
    "    'AUDUSD' :{\n",
    "    'Iterations': 11 ,\n",
    "    'Lr': 0.211,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'EURUSD':{\n",
    "    'Iterations': 23,\n",
    "    'Lr': 0.51,\n",
    "    'depth': 7\n",
    "    },\n",
    "    'EURNZD':{\n",
    "    'Iterations': 188,\n",
    "    'Lr' :0.55,\n",
    "    'depth': 7\n",
    "    }\n",
    "}\n",
    "\n",
    "# with open('parameters', 'wb') as file:\n",
    "#     pickle.dump(parameters, file )\n",
    "    \n",
    "\n",
    "def load_parameters(symbol):\n",
    "    \n",
    "    with open('parameters', 'rb') as file:\n",
    "        parameters = pickle.load(file )\n",
    "        \n",
    "    return parameters[symbol]\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "252f2662-b410-460a-a85b-85ea8d7757dd",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Python Code To Make A Binary File From All Daily OHLC Currency\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "90b8b88b-9629-40ae-82a7-eef672a138c0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:123: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:124: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:125: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:126: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:105: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:106: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:107: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:108: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:109: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:110: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:111: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:112: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:113: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:114: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:115: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:117: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:118: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:119: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:120: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:121: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:122: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY'\n",
    "        ]\n",
    "\n",
    "\n",
    "currency_ohlc = []\n",
    "count = 0\n",
    "for pair in forex_pairs:\n",
    "    daily_path = f'currency_data/{pair}_Daily.csv'\n",
    "    data = return_df_day(daily_path)\n",
    "    data = add_features(data)\n",
    "    data = make_features_lagged( lag_by= 14)\n",
    "\n",
    "    currency_df = { \"day_features\": data, \"symbol\": f\"{pair}\"  }\n",
    "    currency_ohlc.append(currency_df)\n",
    "    count += 1\n",
    "    print(count)\n",
    "    \n",
    "\n",
    "\n",
    "import pickle\n",
    "\n",
    "with open('day_features_data_lagby_14_v3.bin', 'wb') as file:\n",
    "    pickle.dump(currency_ohlc, file)\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b552eed9-e8d0-48e1-8eb7-ec852fe396a2",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New Harmonic Mean Date\n",
      "2013-06-03         NaN\n",
      "2013-06-04         NaN\n",
      "2013-06-05         NaN\n",
      "2013-06-06         NaN\n",
      "2013-06-07         NaN\n",
      "                ...   \n",
      "2024-04-29   -0.002816\n",
      "2024-04-30   -0.002596\n",
      "2024-05-01   -0.002531\n",
      "2024-05-02   -0.002480\n",
      "2024-05-03   -0.002668\n",
      "Name: High, Length: 2837, dtype: float64\n",
      "Old Harmonic Mean Date\n",
      "2013-06-03    0.016441\n",
      "2013-06-04    0.016441\n",
      "2013-06-05    0.016441\n",
      "2013-06-06    0.016441\n",
      "2013-06-07    0.016441\n",
      "                ...   \n",
      "2024-04-29    0.016441\n",
      "2024-04-30    0.016441\n",
      "2024-05-01    0.016441\n",
      "2024-05-02    0.016441\n",
      "2024-05-03    0.016441\n",
      "Name: harmonic_mean_high, Length: 2837, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "forex_pairs = [\n",
    "    'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY'\n",
    "        ]\n",
    "\n",
    "currency_ohlc = []\n",
    "count = 0\n",
    "for pair in forex_pairs:\n",
    "    daily_path = f'currency_data/{pair}_Daily.csv'\n",
    "    data = return_df_day(daily_path)\n",
    "    data = add_features(data)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "894789d1-5631-401d-acc6-3c13ed282822",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# Calculate RSI\n",
    "def calculate_rsi(series, period=10):\n",
    "    delta = series.diff()\n",
    "    gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()\n",
    "    loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()\n",
    "    rs = gain / loss\n",
    "    return 100 - (100 / (1 + rs))\n",
    "\n",
    "\n",
    "def ema( price, period):\n",
    "\n",
    "  price = np.array(price)\n",
    "  alpha = 2 / (period + 1.0)\n",
    "  alpha_reverse = 1 - alpha\n",
    "  data_length = len(price)\n",
    "\n",
    "  power_factors = alpha_reverse ** (np.arange(data_length + 1))\n",
    "  initial_offset = price[0] * power_factors[1:]\n",
    "\n",
    "  scale_factors = 1 / power_factors[:-1]\n",
    "\n",
    "  weight_factor = alpha * alpha_reverse ** (data_length - 1)\n",
    "\n",
    "  weighted_price_data = price * weight_factor * scale_factors\n",
    "  cumulative_sums = weighted_price_data.cumsum()\n",
    "  ema_values = initial_offset + cumulative_sums * scale_factors[::-1]\n",
    "\n",
    "  return ema_values\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def candle_type(o, h, l, c):\n",
    "\n",
    "    diff = abs(c - o)\n",
    "    o1, c1 = np.roll(o, 1), np.roll(c, 1)  #\n",
    "    min_oc = np.where(o < c, o, c)\n",
    "    max_oc = np.where(o > c, o, c)\n",
    "\n",
    "    pattern = np.where(\n",
    "      np.logical_and( min_oc - l > diff, h - max_oc < diff), 6,\n",
    "      np.where(np.logical_and( h - max_oc > diff, min_oc - l < diff),\n",
    "      4, np.where(np.logical_and(np.logical_and(c > o, c1 < o1), np.logical_and(c > o1, o < c1)),\n",
    "        5, np.where( min_oc - l > diff, 3,\n",
    "                      np.where(np.logical_and( h - max_oc > diff,\n",
    "                  min_oc - l < diff),\n",
    "                      2, np.where(np.logical_and(np.logical_and(c > o, c1 < o1), np.logical_and(c > o1, o < c1)),\n",
    "                      1, 0))))))\n",
    "    return pattern\n",
    "\n",
    "\n",
    "def heikin_ashi_status( ha_open, ha_close):\n",
    "\n",
    "    candles = np.full_like(ha_close, '', dtype='U10')\n",
    "\n",
    "    for i in range(1, len(ha_close)):\n",
    "\n",
    "        if ha_close[i] > ha_open[i]: candles[i] = 2 #'Green'\n",
    "\n",
    "        elif ha_close[i] < ha_open[i]: candles[i] = 1 # 'Red'\n",
    "\n",
    "        else: candles[i] = 0 #'Neutral'\n",
    "\n",
    "    return candles\n",
    "\n",
    "def heikin_ashi_candles( open, high, low, close):\n",
    "\n",
    "    ha_low, ha_close = np.empty(len(close), dtype=np.float32), np.empty(len(close), dtype=np.float32)\n",
    "    ha_open, ha_high = np.empty(len(close), dtype=np.float32), np.empty(len(close), dtype=np.float32)\n",
    "\n",
    "    ha_open[0] = (open[0] + close[0]) / 2\n",
    "    ha_close[0] = (close[0] + open[0] + high[0] + low[0]) / 4\n",
    "\n",
    "    for i in range(1, len(close)):\n",
    "        ha_open[i] = (ha_open[i - 1] + ha_close[i - 1]) / 2\n",
    "        ha_close[i] = (open[i] + high[i] + low[i] + close[i]) / 4\n",
    "        ha_high[i] = max(high[i], ha_open[i], ha_close[i])\n",
    "        ha_low[i] = min(low[i], ha_open[i], ha_close[i])\n",
    "\n",
    "    return ha_open, ha_close, ha_high, ha_low\n",
    "\n",
    "\n",
    "def true_range( high, low, close):\n",
    "\n",
    "  close_shift = shift(close, 1)\n",
    "  high_low, high_close, low_close = np.array(high - low, dtype=np.float32), \\\n",
    "                                    np.array(abs(high - close_shift), dtype=np.float32), \\\n",
    "                                    np.array(abs(low - close_shift), dtype=np.float32)\n",
    "\n",
    "  true_range = np.max(np.hstack((high_low, high_close, low_close)).reshape(-1, 3), axis=1)\n",
    "\n",
    "  return true_range\n",
    "\n",
    "\n",
    "\n",
    "def shift(array, place):\n",
    "\n",
    "  array = np.array(array, dtype=np.float32)\n",
    "  shifted = np.roll(array, place)\n",
    "  shifted[0:place] = np.nan\n",
    "  shifted[np.isnan(shifted)] = np.nanmean(shifted)\n",
    "\n",
    "  return shifted\n",
    "\n",
    "\n",
    "def ma_based_supertrend_indicator( high, low, close, atr_length=10, atr_multiplier=3, ma_length=10):\n",
    "\n",
    "    # Calculate True Range and Smoothed ATR\n",
    "    tr = true_range(high, low, close)\n",
    "    atr = ema(tr, atr_length)\n",
    "\n",
    "    upper_band = (high + low) / 2 + (atr_multiplier * atr)\n",
    "    lower_band = (high + low) / 2 - (atr_multiplier * atr)\n",
    "\n",
    "    trend = np.zeros(len(atr))\n",
    "\n",
    "    # Calculate Moving Average\n",
    "    ema_values = ema(close, ma_length)\n",
    "\n",
    "    if ema_values[0] > lower_band[0]:\n",
    "        trend[0] = lower_band[0]\n",
    "    elif ema_values[0] < upper_band[0]:\n",
    "        trend[0] = upper_band[0]\n",
    "    else:\n",
    "        trend[0] = upper_band[0]\n",
    "\n",
    "    # Compute final upper and lower bands\n",
    "    for i in range(1, len(close)):\n",
    "        if ema_values[i] > trend[i - 1]:\n",
    "            trend[i] = max(trend[i - 1], lower_band[i])\n",
    "\n",
    "\n",
    "        elif ema_values[i] < trend[i - 1]:\n",
    "            trend[i] = min(trend[i - 1], upper_band[i])\n",
    "\n",
    "        else:\n",
    "            trend[i] = trend[i - 1]\n",
    "\n",
    "    status_value = np.where(ema_values > trend, 1.0, -1.0)\n",
    "\n",
    "    return trend, status_value\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def supertrend_status_crossover( status_value):\n",
    "\n",
    "\n",
    "    prev_status = np.roll(status_value, 1)\n",
    "    supertrend_status_crossover = np.where((prev_status < 0) & (status_value > 0), 1.0, np.where((prev_status > 0) & (status_value < 0), -1.0, 0))\n",
    "\n",
    "    return supertrend_status_crossover\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def supertrend_indicator(high, low, close, period, multiplier=1.0):\n",
    "\n",
    "    true_range_value = true_range(high, low, close)\n",
    "\n",
    "    smoothed_atr = ema(true_range_value, period)\n",
    "\n",
    "    upper_band = (high + low) / 2 + (multiplier * smoothed_atr)\n",
    "    lower_band = (high + low) / 2 - (multiplier * smoothed_atr)\n",
    "\n",
    "    supertrend = np.zeros(len(true_range_value))\n",
    "    trend = np.zeros(len(true_range_value))\n",
    "\n",
    "    if close[0] > upper_band[0]: trend[0] = upper_band[0]\n",
    "    elif close[0] < lower_band[0]: trend[0] = lower_band[0]\n",
    "    else:  trend[0] = upper_band[0]\n",
    "\n",
    "    for i in range(1, len(close)):\n",
    "\n",
    "        if close[i] > upper_band[i]: trend[i] = upper_band[i]\n",
    "        elif close[i] < lower_band[i]: trend[i] = lower_band[i]\n",
    "        else: trend[i] = trend[i - 1]\n",
    "\n",
    "    # Calculate Buy/Sell Signals using numpy where  # np.where( close > trend, '1 Buy', '-1 Sell')\n",
    "    status_value = np.where(close > trend, 1.0, -1.0)\n",
    "\n",
    "    return trend, status_value\n",
    "\n",
    "def supertrend_status_crossover(status_value):\n",
    "\n",
    "    prev_status = np.roll(status_value, 1)\n",
    "    supertrend_status_crossover = np.where((prev_status < 0) & (status_value > 0), 1.0, np.where((prev_status > 0) & (status_value < 0), -1.0, 0))\n",
    "\n",
    "    return supertrend_status_crossover\n",
    "\n",
    "\n",
    "def rsi_crossover(smoothed_rsi):\n",
    "    \n",
    "    prev_signal = np.roll(smoothed_rsi, 1)\n",
    "    direction, crossover = direction_crossover_signal_line(smoothed_rsi, prev_signal)\n",
    "    \n",
    "    return crossover\n",
    "    pass\n",
    "\n",
    "def rsi_crossover_with_sma(rsi , slow_smoothed_rsi): # 10 period sma of actual rsi\n",
    "    \n",
    "    direction = np.where(rsi - slow_smoothed_rsi > 0, 1, -1)\n",
    "    \n",
    "    prev_direction = np.roll(direction, 1)\n",
    "    crossover = np.where((prev_direction == -1) & (direction == 1), 1,\n",
    "                             np.where((prev_direction == 1) & (direction == -1), -1, 0))\n",
    "    \n",
    "    return crossover\n",
    "\n",
    "def calculate_harmonic_mean(series, period=3):\n",
    "    def harmonic_mean(s):\n",
    "        n = len(s)\n",
    "        return n / sum(1 / x for x in s if x != 0) if n > 0 else 0\n",
    "    \n",
    "    return series.rolling(window=period).apply(harmonic_mean, raw=True)\n",
    "\n",
    "\n",
    "def calculate_hm_high(series, period=3):\n",
    "\n",
    "    def harmonic_mean_high(high):\n",
    "        n = len(high)\n",
    "        prev_high = np.roll(high, 1)    \n",
    "        diff = high - prev_high\n",
    "\n",
    "        hm = n / sum(1 / x for x in diff if x != 0)\n",
    "        y = np.sin(hm)\n",
    "\n",
    "        y_pi = y * np.pi\n",
    "        return y_pi\n",
    "    return  series.rolling(window=period).apply(harmonic_mean_high, raw=True)\n",
    "\n",
    "def harmonic_mean_high(high):\n",
    "        n = len(high)\n",
    "        prev_high = np.roll(high, 1)    \n",
    "        diff = high - prev_high\n",
    "\n",
    "        hm = n / sum(1 / x for x in diff if x != 0)\n",
    "        y = np.sin(hm)\n",
    "\n",
    "        y_pi = y * np.pi\n",
    "        return y_pi\n",
    "\n",
    "\n",
    "def calculate_hm_low(series, period=3):\n",
    "\n",
    "    def harmonic_mean_low(low):\n",
    "        n = len(low)\n",
    "        prev_low = np.roll(low, 1)    \n",
    "        diff = prev_low - low\n",
    "\n",
    "        hm = n / sum(1 / x for x in diff if x != 0)\n",
    "        y = np.sin(hm)\n",
    "\n",
    "        y_pi = y * np.pi\n",
    "        return y_pi\n",
    "    return  series.rolling(window=period).apply(harmonic_mean_low, raw=True)\n",
    "    \n",
    "\n",
    "def direction_crossover_signal_line(signal, prev_signal):\n",
    "\n",
    "        direction = np.where(signal - prev_signal > 0, 1, -1) # current bigger then upward direction , if current small then downward direction\n",
    "        prev_direction = np.roll(direction, 1)\n",
    "        crossover = np.where((prev_direction == -1) & (direction == 1), 1,\n",
    "                             np.where((prev_direction == 1) & (direction == -1), -1, 0))\n",
    "\n",
    "        return direction, crossover"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc761857-01d2-47c4-8dd9-903c4a9d55e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def add_features(data):\n",
    "    \n",
    "    open_value, high, low, close = data['Open'], data['High'], data['Low'], data['Close']\n",
    "    # high , low = data['High'].to_numpy(), data['Low'].to_numpy()\n",
    "    \n",
    "#     elastic_supertrend, es_status_value = ma_based_supertrend_indicator( high, low, close, atr_length=10, atr_multiplier=2.5, ma_length=10)\n",
    "\n",
    "#     elastic_supertrend_crossover = supertrend_status_crossover(es_status_value)\n",
    "\n",
    "    # supertrend, supertrend_status_value = supertrend_indicator(high, low, close, period= 10, multiplier=0.66)\n",
    "    # supertrend_crossover = supertrend_status_crossover(supertrend_status_value)\n",
    "\n",
    "    candle_type_value  = candle_type(open_value, high, low, close)\n",
    "\n",
    "    # ha_open, ha_close, ha_high, ha_low = heikin_ashi_candles(open_value, high, low, close)\n",
    "    # heikin_ashi_candle = heikin_ashi_status(ha_open, ha_close)\n",
    "    hl2 = ( data['High'] + data['Low'] ) / 2\n",
    "    \n",
    "    rsi  = calculate_rsi(hl2, period=10)\n",
    "    smoothed_rsi =  rsi.rolling(window=3).mean()\n",
    "    slow_smoothed_rsi =  rsi.rolling(window=10).mean()\n",
    "    \n",
    "    data['rsi_sma_fast'] = smoothed_rsi\n",
    "    data['rsi_sma_slow'] = slow_smoothed_rsi\n",
    "    \n",
    "    data['rsi'] =  rsi\n",
    "    data['rsi_crossover_fast'] =  rsi_crossover(smoothed_rsi)\n",
    "    data['rsi_crossover_slow'] =  rsi_crossover_with_sma(rsi , slow_smoothed_rsi)\n",
    "    \n",
    "    \n",
    "    data['STDEV'] = data['Close'].rolling(window=5).std()\n",
    "    data['Upper_Band'] = data['Close'].rolling(window=5).mean() + (data['Close'].rolling(window=5).std() * 2)\n",
    "    data['Lower_Band'] = data['Close'].rolling(window=5).mean() - (data['Close'].rolling(window=5).std() * 2)\n",
    "\n",
    "    data['candle_type'] = candle_type_value\n",
    "    \n",
    "    data['slow_harmonic_mean'] = calculate_harmonic_mean(data['Close'], period=9)\n",
    "    data['fast_harmonic_mean'] = calculate_harmonic_mean(data['Close'], period=3)\n",
    "    \n",
    "    data['harmonic_mean_high'] = calculate_hm_high(high)\n",
    "    # data['harmonic_mean_high'] = harmonic_mean_high(high)\n",
    "    # print(\"New Harmonic Mean\",calculate_hm_high(high)[:-20])\n",
    "    # print(\"Old Harmonic Mean\",data['harmonic_mean_high'][:-20])\n",
    "    \n",
    "    data['harmonic_mean_low']  = calculate_hm_low(low)\n",
    "\n",
    "    data['Price_Range'] = data['High'] - data['Low']\n",
    "    data['Median_Price'] = (data['High'] + data['Low']) / 2\n",
    "    \n",
    "    data['daily_returns'] = data['Close'] - data['Close'].shift(1)\n",
    "    \n",
    "    data['Day_of_Week'] = data.index.dayofweek + 1  # Monday=1, ..., Friday=5\n",
    "    data['Week_of_Month'] = (data.index.day - 1) // 7 + 1\n",
    "    data['Month'] = data.index.month\n",
    "\n",
    "    return data\n",
    "\n",
    "#     data['supertrend']  = np.asarray(supertrend)\n",
    "#     data['supertrend_status']  = np.asarray(supertrend_status_value)\n",
    "\n",
    "\n",
    "#     data['supertrend_crossover'] = np.asarray(supertrend_crossover)\n",
    "#     data['supertrend_value'] = np.asarray(supertrend)\n",
    "\n",
    "#     data['elastic_supertrend'] = elastic_supertrend\n",
    "#     data['elastic_supertrend_status'] = es_status_value\n",
    "#     data['elastic_supertrend_cross'] = elastic_supertrend_crossover\n",
    "\n",
    "#     data['heikin_ashi'] = heikin_ashi_candle\n",
    "\n",
    "#     ema_3  = ema( (high+low/2), 3 )\n",
    "#     ema_5  = ema( (high+low/2), 5 )\n",
    "#     ema_7  = ema( (high+low/2), 7 )\n",
    "#     ema_14 = ema( (high+low/2), 14)\n",
    "\n",
    "#     # Add seasonality features\n",
    "#     data['Day_of_Week'] = data.index.dayofweek + 1  # Monday=1, ..., Friday=5\n",
    "#     data['Week_of_Month'] = (data.index.day - 1) // 7 + 1\n",
    "#     data['Month'] = data.index.month\n",
    "\n",
    "    # Add numerical features\n",
    "    # data['Prev_Close'] = data['Close'].shift(1)\n",
    "    \n",
    "    \n",
    "    \n",
    "#     data['ema_3']  = ema_3\n",
    "#     data['ema_5']  = ema_5\n",
    "#     data['ema_7']  = ema_7\n",
    "#     data['ema_14'] = ema_14\n",
    "\n",
    "#     selected_columns = data[['ema_3', 'ema_5', 'ema_7' ]]\n",
    "\n",
    "#     ema_mean = np.mean(selected_columns, axis=1)\n",
    "#     data['ema_difference'] = pd.Series(np.subtract( ema_mean, ema_14 ))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def make_features_lagged( lag_by= 9):\n",
    "    \n",
    "    # Create window-based features\n",
    "    window_size = lag_by\n",
    "\n",
    "    for i in range(1, window_size + 1):\n",
    "        data[f'Day_of_Week_T-{i}']   = data['Day_of_Week'].shift(i)\n",
    "        data[f'Week_of_Month_T-{i}'] = data['Week_of_Month'].shift(i)\n",
    "        data[f'Month_T-{i}'] = data['Month'].shift(i)\n",
    "        data[f'Close_T-{i}'] = data['Close'].shift(i)\n",
    "        data[f'Open_T-{i}'] = data['Open'].shift(i)\n",
    "        data[f'High_T-{i}'] = data['High'].shift(i)\n",
    "        data[f'Low_T-{i}']  = data['Low'].shift(i)\n",
    "        data[f'STDEV_T-{i}']   = data['STDEV'].shift(i)\n",
    "        data[f'rsi_T-{i}']  = data['rsi'].shift(i)\n",
    "        data[f'rsi_crossover_fast_T-{i}']  = data['rsi_crossover_fast'].shift(i)\n",
    "        data[f'rsi_crossover_slow_T-{i}']  = data['rsi_crossover_slow'].shift(i)\n",
    "        \n",
    "        data[f'Price_Range_T-{i}']   = data['Price_Range'] .shift(i)\n",
    "        data[f'Median_Price_T-{i}']  = data['Median_Price'].shift(i)\n",
    "        data[f'Upper_Band_T-{i}']  = data['Upper_Band'].shift(i)\n",
    "        data[f'Lower_Band_T-{i}']  = data['Lower_Band'].shift(i)\n",
    "        data[f'candle_type_T-{i}'] = data['candle_type'].shift(i)\n",
    "        data[f'slow_harmonic_mean_T-{i}'] = data['slow_harmonic_mean'].shift(i)\n",
    "        data[f'fast_harmonic_mean_T-{i}'] = data['fast_harmonic_mean'].shift(i)\n",
    "        data[f'harmonic_mean_high_T-{i}'] = data['harmonic_mean_high'].shift(i)\n",
    "        data[f'harmonic_mean_low_T-{i}'] = data['harmonic_mean_low'].shift(i)\n",
    "        data[f'daily_returns_T-{i}'] = data['daily_returns'].shift(i)\n",
    "\n",
    "    return data\n",
    "        \n",
    "        \n",
    "#         data[f'supertrend_crossover_T-{i}'] = data['supertrend_crossover'].shift(i)\n",
    "#         data[f'elastic_supertrend_T-{i}']   = data['elastic_supertrend'].shift(i)\n",
    "#         data[f'elastic_supertrend_status_T-{i}']   = data['elastic_supertrend_status'].shift(i)\n",
    "\n",
    "#         data[f'elastic_supertrend_cross_T-{i}'] = data['elastic_supertrend_cross'].shift(i)\n",
    "\n",
    "\n",
    "        # data[f'ema_3_T-{i}']  = data['ema_3'].shift(i)\n",
    "        # data[f'ema_5_T-{i}']  = data['ema_5'].shift(i)\n",
    "        # data[f'ema_7_T-{i}']  = data['ema_7'].shift(i)\n",
    "        # data[f'ema_14_T-{i}'] = data['ema_14'].shift(i)\n",
    "        # data[f'ema_difference_T-{i}'] = data['ema_difference'].shift(i)\n",
    "        # data[f'supertrend_value_T-{i}'] = data['supertrend_value'].shift(i)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def return_df_day(daily_path):\n",
    "    \n",
    "    data = pd.read_csv(daily_path, sep='\\t')\n",
    "\n",
    "    # Rename columns as requested\n",
    "    data.rename(columns={\n",
    "    '<DATE>': 'Date',\n",
    "    '<OPEN>': 'Open',\n",
    "    '<HIGH>': 'High',\n",
    "    '<LOW>': 'Low',\n",
    "    '<CLOSE>': 'Close',\n",
    "    '<TICKVOL>': 'TickVol',\n",
    "    '<VOL>': 'Vol',\n",
    "    '<SPREAD>': 'Spread'\n",
    "    }, inplace=True)\n",
    "\n",
    "    data['Date'] = pd.to_datetime(data['Date'], format='%Y.%m.%d')\n",
    "\n",
    "    # Set Date column as the index\n",
    "    data.set_index('Date', inplace=True)\n",
    "\n",
    "    data = data[['Open', 'High', 'Low', 'Close']]\n",
    "\n",
    "    return data\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04e37fc9-7da2-4fa9-9ab0-b6404763b9bf",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Python Code To Make A Binary File From All Hour4 OHLC Currency\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ef4b3bbb-9656-480e-8b07-c953fa6a0c24",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:115: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:123: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:124: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:141: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:154: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:156: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:158: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:168: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:184: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:86: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:87: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:89: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:90: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:96: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:97: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:98: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:99: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/ipykernel_launcher.py:100: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead.  To get a de-fragmented frame, use `newframe = frame.copy()`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "forex_pairs = [ 'AUDCAD', 'AUDCHF', 'AUDJPY', 'AUDNZD', 'AUDUSD',\n",
    "    'CADCHF', 'CADJPY',\n",
    "    'CHFJPY', \n",
    "    'EURAUD', 'EURCAD', 'EURCHF', 'EURGBP', \n",
    "    'EURJPY', 'EURNZD', 'EURUSD',\n",
    "    'GBPAUD', 'GBPCAD', 'GBPCHF', \n",
    "    'GBPJPY', 'GBPUSD', 'GBPNZD',\n",
    "    'NZDCAD', 'NZDCHF', 'NZDJPY', 'NZDUSD',  \n",
    "    'USDCHF', 'USDCAD', 'USDJPY' ]\n",
    "\n",
    "\n",
    "currency_ohlc = []\n",
    "count = 0\n",
    "for pair in forex_pairs:\n",
    "    # daily_path = f'currency_data/{pair[:6]}_Daily.csv'\n",
    "    hour4_path = f'currency_data/{pair}_H4.csv'\n",
    "\n",
    "    data_h4     = rename_h4_df(hour4_path)\n",
    "\n",
    "    reshaped_h4 = combine_ohlc_into_single_day(data_h4)\n",
    "\n",
    "    features_h4 = add_ohlc_in_lagged(reshaped_h4,  lag_by= 3 )\n",
    "\n",
    "    features_h4 = add_features(features_h4,  lag_by= 3)\n",
    "    \n",
    "    currency_df = { \"hour4_features\": features_h4, \"symbol\": f\"{pair}\"  }\n",
    "    \n",
    "    currency_ohlc.append(currency_df)\n",
    "    count += 1\n",
    "    print(count)\n",
    "\n",
    "print(count)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "import pickle\n",
    "\n",
    "with open('hour4_features_data_lag_by_3_v2.bin', 'wb') as file:\n",
    "    pickle.dump(currency_ohlc, file)\n",
    "    \n",
    "\n",
    "# import pickle\n",
    "\n",
    "# with open('hour4_less_features_data_lag_by_3.bin', 'wb') as file:\n",
    "#     pickle.dump(currency_ohlc, file)\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c46cf20e-a73f-4f52-977b-196a2087fca5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# WITH LESS FEATURES FOR HOUR 4 \n",
    "\n",
    "\n",
    "def rename_h4_df(hour4_path):\n",
    "\n",
    "    data_h4 = pd.read_csv(hour4_path, sep='\\t')\n",
    "\n",
    "    # Rename columns as requested\n",
    "    data_h4.rename(columns={\n",
    "        '<DATE>': 'Date',\n",
    "        '<OPEN>': 'Open_h4',\n",
    "        '<HIGH>': 'High_h4',\n",
    "        '<LOW>': 'Low_h4',\n",
    "        '<CLOSE>': 'Close_h4',\n",
    "        '<TICKVOL>': 'TickVol',\n",
    "        '<VOL>': 'Vol',\n",
    "        '<SPREAD>': 'Spread'\n",
    "    }, inplace=True)\n",
    "\n",
    "    data_h4['Date'] = pd.to_datetime(data_h4['Date'], format='%Y.%m.%d')\n",
    "\n",
    "    # Set Date column as the index\n",
    "    data_h4.set_index('Date', inplace=True)\n",
    "\n",
    "    data_h4 = data_h4[['Open_h4', 'High_h4', 'Low_h4', 'Close_h4']]\n",
    "\n",
    "    return data_h4\n",
    "\n",
    "\n",
    "def combine_ohlc_into_single_day(data_h4):\n",
    "    grouped = data_h4.groupby(data_h4.index.date)\n",
    "\n",
    "    # Create a new dataframe to store the result\n",
    "    reshaped_h4 = pd.DataFrame()\n",
    "\n",
    "    # Extract Open, High, Low, Close for each 4-hour period and reshape\n",
    "    for date, group in grouped:\n",
    "        group = group.reset_index(drop=True)\n",
    "        for i in range(0, len(group)):\n",
    "            if i == 0:\n",
    "                reshaped_h4.at[date, f'Open_h4_{i}'] = group.loc[i, 'Open_h4']\n",
    "                reshaped_h4.at[date, f'High_h4_{i}'] = group.loc[i, 'High_h4']\n",
    "                reshaped_h4.at[date, f'Low_h4_{i}'] = group.loc[i, 'Low_h4']\n",
    "                reshaped_h4.at[date, f'Close_h4_{i}'] = group.loc[i, 'Close_h4']\n",
    "            else:\n",
    "                reshaped_h4.at[date, f'Open_h4_{i}'] = group.loc[i, 'Open_h4']\n",
    "                reshaped_h4.at[date, f'High_h4_{i}'] = group.loc[i, 'High_h4']\n",
    "                reshaped_h4.at[date, f'Low_h4_{i}'] = group.loc[i, 'Low_h4']\n",
    "                reshaped_h4.at[date, f'Close_h4_{i}'] = group.loc[i, 'Close_h4']\n",
    "\n",
    "    return reshaped_h4\n",
    "    \n",
    "\n",
    "def add_ohlc_in_lagged(reshaped_h4, lag_by= 7):\n",
    "    \n",
    "    features_h4 = pd.DataFrame()\n",
    "    for candles in range(0, 6): # 0 --> 5 all 6 candles\n",
    "        for day in range(1, lag_by + 1): # last 3 days = 6 * 3 = Last H4 18 candles\n",
    " \n",
    "            # new name will be candle number and day shifted from\n",
    "            features_h4[f'Close_h4_{candles}_T-{day}'] = reshaped_h4[f'Close_h4_{candles}'].shift(day)\n",
    "            features_h4[f'High_h4_{candles}_T-{day}']  = reshaped_h4[f'High_h4_{candles}'].shift(day)\n",
    "            features_h4[f'Open_h4_{candles}_T-{day}']  = reshaped_h4[f'Open_h4_{candles}'].shift(day)\n",
    "            features_h4[f'Low_h4_{candles}_T-{day}']  = reshaped_h4[f'Low_h4_{candles}'].shift(day)\n",
    "    \n",
    "    return  features_h4\n",
    "\n",
    "\n",
    "def add_features(features_h4, lag_by= 7):\n",
    "    \n",
    "    # features_h4.fillna(0.0, inplace= True)\n",
    "    features_h4 = features_h4.apply(lambda x: x.fillna(x.mean()), axis=0)\n",
    "    \n",
    "    for candles in range(0, 6): # 0 --> 5 all 6 candles\n",
    "        for day in range(1, lag_by + 1): # last 3 days = 6 * 3 = Last H4 18 candles\n",
    "            pass \n",
    "            # open_value  = features_h4[f'Open_h4_{candles}_T-{day}']\n",
    "            close_value = features_h4[f'Close_h4_{candles}_T-{day}']\n",
    "            high_value = features_h4[f'High_h4_{candles}_T-{day}']\n",
    "            low_value  = features_h4[f'Low_h4_{candles}_T-{day}']\n",
    "            hl2 = ( high_value + low_value ) / 2\n",
    "\n",
    "            # features_h4[f'Low_h4_{candles}_T-{day}'].fillna(features_h4[f'Low_h4_{candles}_T-{day}'].mean(), inplace=True)\n",
    "            # features_h4[f'High_h4_{candles}_T-{day}'].fillna(features_h4[f'High_h4_{candles}_T-{day}'].mean(), inplace=True)\n",
    "            # features_h4[f'Close_h4_{candles}_T-{day}'].fillna(features_h4[f'Close_h4_{candles}_T-{day}'].mean(), inplace=True)\n",
    "            features_h4[f'slow_harmonic_mean_{candles}_T-{day}'] = calculate_harmonic_mean(close_value , period=27)\n",
    "            features_h4[f'fast_harmonic_mean_{candles}_T-{day}'] = calculate_harmonic_mean(close_value, period=9)\n",
    "    \n",
    "            features_h4[f'harmonic_mean_high_{candles}_T-{day}'] = calculate_hm_high(high_value)\n",
    "            features_h4[f'harmonic_mean_low_{candles}_T-{day}']  = calculate_hm_low(low_value)\n",
    "            \n",
    "            rsi  = calculate_rsi(hl2, period=27)\n",
    "            smoothed_rsi =  rsi.rolling(window=9).mean()\n",
    "            slow_smoothed_rsi =  rsi.rolling(window=10).mean()\n",
    "\n",
    "            features_h4[f'rsi_sma_fast_{candles}_T-{day}'] = smoothed_rsi\n",
    "            features_h4[f'rsi_sma_slow_{candles}_T-{day}'] = slow_smoothed_rsi\n",
    "            features_h4[f'rsi_{candles}_T-{day}'] =  rsi\n",
    "            features_h4[f'rsi_crossover_fast_{candles}_T-{day}'] =  rsi_crossover(smoothed_rsi)\n",
    "            features_h4[f'rsi_crossover_slow_{candles}_T-{day}'] =  rsi_crossover_with_sma(rsi , slow_smoothed_rsi)\n",
    "            \n",
    "            \n",
    "            open_value  = features_h4[f'Open_h4_{candles}_T-{day}'].values\n",
    "            close_value = features_h4[f'Close_h4_{candles}_T-{day}'].values\n",
    "            high_value = features_h4[f'High_h4_{candles}_T-{day}'].values\n",
    "            low_value  = features_h4[f'Low_h4_{candles}_T-{day}'].values\n",
    "\n",
    "            open_val  = features_h4[f'Open_h4_{candles}_T-{day}']\n",
    "            close_val = features_h4[f'Close_h4_{candles}_T-{day}']\n",
    "            high_val = features_h4[f'High_h4_{candles}_T-{day}']\n",
    "            low_val  = features_h4[f'Low_h4_{candles}_T-{day}']\n",
    "\n",
    "            hlc = ( features_h4[f'High_h4_{candles}_T-{day}'] + features_h4[f'Low_h4_{candles}_T-{day}'] + features_h4[f'Close_h4_{candles}_T-{day}']) / 3 \n",
    "\n",
    "            features_h4[f'true_range_h4_{candles}_T-{day}'] = pd.Series(high_value- low_value)\n",
    "\n",
    "            # features_h4[f'median_h4_{candles}_T-{day}'] = pd.Series( (high_value + low_value) / 2 )\n",
    "            \n",
    "            \n",
    "#             features_h4[f'RSI_slow_{candles}_T-{day}']   = calculate_rsi(close_val, period=21)\n",
    "#             features_h4[f'RSI_fast_{candles}_T-{day}']   = calculate_rsi(close_val, period=9)\n",
    "            \n",
    "            features_h4[f'stdev_slow_{candles}_T-{day}'] = close_val.rolling(window=18).std()\n",
    "            features_h4[f'stdev_fast_{candles}_T-{day}'] = close_val.rolling(window=9).std()\n",
    "    \n",
    "#             features_h4[f'Upper_Band_slow_{candles}_T-{day}'] = close_val.rolling(window=18).mean() + (close_val.rolling(window=5).std() * 2)\n",
    "#             features_h4[f'Lower_Band_slow_{candles}_T-{day}'] = close_val.rolling(window=18).mean() - (close_val.rolling(window=5).std() * 2)\n",
    "\n",
    "#             features_h4[f'Upper_Band_fast_{candles}_T-{day}'] = close_val.rolling(window=9).mean() + (close_val.rolling(window=5).std() * 2)\n",
    "#             features_h4[f'Lower_Band_fast_{candles}_T-{day}'] = close_val.rolling(window=9).mean() - (close_val.rolling(window=5).std() * 2)\n",
    "\n",
    "#             ema3_value  = hlc.ewm(span=3, adjust=False).mean()\n",
    "#             ema5_value  = hlc.ewm(span=5, adjust=False).mean()\n",
    "#             ema7_value  = hlc.ewm(span=7, adjust=False).mean()\n",
    "#             ema14_value = hlc.ewm(span=14, adjust=False).mean()\n",
    "\n",
    "            highest_high = features_h4[f'High_h4_{candles}_T-{day}'].rolling(window=9).max()\n",
    "            lowest_low   = features_h4[f'Low_h4_{candles}_T-{day}'].rolling(window=9).min()\n",
    "\n",
    "            # Calculate the relative range\n",
    "            features_h4[f'relative_range_h4_{candles}_T-{day}'] = close_value - ( ( highest_high + lowest_low ) / 2)\n",
    "\n",
    "\n",
    "            candle_type_value  = candle_type(open_value, high_value, low_value, close_value)\n",
    "\n",
    "\n",
    "#             elastic_supertrend, es_status_value = ma_based_supertrend_indicator( high_value, low_value, close_value, atr_length=9, atr_multiplier=2.5, ma_length=9)\n",
    "\n",
    "#             elastic_supertrend_crossover = supertrend_status_crossover(es_status_value)\n",
    "\n",
    "            supertrend, supertrend_status_value = supertrend_indicator(high_value, low_value, close_value, period= 27, multiplier=5.5)\n",
    "            supertrend_crossover = supertrend_status_crossover(supertrend_status_value)\n",
    "\n",
    "            features_h4[f'supertrend_h4_{candles}_T-{day}'] = supertrend\n",
    "\n",
    "            features_h4[f'supertrend_status_h4_{candles}_T-{day}'] = supertrend_status_value\n",
    "\n",
    "            features_h4[f'supertrend_crossover_h4_{candles}_T-{day}'] = supertrend_crossover\n",
    "\n",
    "\n",
    "#             features_h4[f'es_supertrend_h4_{candles}_T-{day}'] = elastic_supertrend\n",
    "\n",
    "#             features_h4[f'es_supertrend_crossover_h4_{candles}_T-{day}'] = elastic_supertrend_crossover\n",
    "\n",
    "#             features_h4[f'es_supertrend_status_h4_{candles}_T-{day}'] = es_status_value\n",
    "\n",
    "            \n",
    "            features_h4[f'candle_type_h4_{candles}_T-{day}'] = candle_type_value\n",
    "\n",
    "            # features_h4[f'smi_direct_h4_{candles}_T-{day}']  = smi_fast_direction\n",
    "\n",
    "            # features_h4[f'smi_h4_{candles}_T-{day}'] = smi_fast\n",
    "\n",
    "#             features_h4[f'ema3_h4_{candles}_T-{day}']  = pd.Series(ema3_value)\n",
    "\n",
    "#             features_h4[f'ema5_h4_{candles}_T-{day}']  = pd.Series(ema5_value)\n",
    "\n",
    "#             features_h4[f'ema7_h4_{candles}_T-{day}']  = pd.Series(ema7_value)\n",
    "\n",
    "#             features_h4[f'ema14_h4_{candles}_T-{day}'] = pd.Series(ema14_value)\n",
    "\n",
    "            ha_open, ha_close, ha_high, ha_low = heikin_ashi_candles(open_value, high_value, low_value, close_value)\n",
    "            heikin_ashi_candle = heikin_ashi_status(ha_open, ha_close)\n",
    "            features_h4[f'heikin_ashi_{candles}_T-{day}'] = heikin_ashi_candle\n",
    "\n",
    "#             selected_columns = features_h4[[ema3_column, ema5_column, ema7_column ]]\n",
    "\n",
    "#             ema_mean = np.mean(selected_columns, axis=1)\n",
    "#             features_h4[f'ema_difference_{candles}_T-{day}'] = pd.Series(np.subtract( ema_mean, ema14_value ))\n",
    "\n",
    "        \n",
    "    return  features_h4\n",
    "\n",
    "\n",
    "\n",
    "def ema( price, period):\n",
    "\n",
    "  price = np.array(price)\n",
    "  alpha = 2 / (period + 1.0)\n",
    "  alpha_reverse = 1 - alpha\n",
    "  data_length = len(price)\n",
    "\n",
    "  power_factors = alpha_reverse ** (np.arange(data_length + 1))\n",
    "  initial_offset = price[0] * power_factors[1:]\n",
    "\n",
    "  scale_factors = 1 / power_factors[:-1]\n",
    "\n",
    "  weight_factor = alpha * alpha_reverse ** (data_length - 1)\n",
    "\n",
    "  weighted_price_data = price * weight_factor * scale_factors\n",
    "  cumulative_sums = weighted_price_data.cumsum()\n",
    "  ema_values = initial_offset + cumulative_sums * scale_factors[::-1]\n",
    "\n",
    "  return ema_values\n",
    "    \n",
    "\n",
    "\n",
    "# Calculate RSI\n",
    "def calculate_rsi(series, period=5):\n",
    "    delta = series.diff()\n",
    "    gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()\n",
    "    loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()\n",
    "    rs = gain / loss\n",
    "    return 100 - (100 / (1 + rs))\n",
    "\n",
    "    \n",
    "def moving_max(array, window_size):\n",
    "   \n",
    "    rolling_max = np.full(array.shape, 0.0)\n",
    "    \n",
    "    for i in range(len(array) - window_size + 1):\n",
    "        window_values = array[i:i + window_size]\n",
    "        rolling_max[i + window_size - 1] = np.max(window_values)\n",
    "        \n",
    "    rolling_max[np.isnan(rolling_max)] = np.nanmean(rolling_max)\n",
    "    return rolling_max    \n",
    "    \n",
    "\n",
    "def moving_min(array, window_size):\n",
    "\n",
    "    rolling_min = np.full(array.shape, 0.0)\n",
    "    for i in range(len(array) - window_size + 1):\n",
    "        window_values = array[i:i + window_size]\n",
    "        rolling_min[i + window_size - 1] = np.min(window_values)\n",
    "        \n",
    "    rolling_min[np.isnan(rolling_min)] = np.nanmean(rolling_min)\n",
    "    return rolling_min\n",
    "\n",
    "\n",
    "\n",
    "def true_range( high, low, close):\n",
    "\n",
    "  close_shift = shift(close, 1)\n",
    "  high_low, high_close, low_close = np.array(high - low, dtype=np.float32), \\\n",
    "                                    np.array(abs(high - close_shift), dtype=np.float32), \\\n",
    "                                    np.array(abs(low - close_shift), dtype=np.float32)\n",
    "\n",
    "  true_range = np.max(np.hstack((high_low, high_close, low_close)).reshape(-1, 3), axis=1)\n",
    "\n",
    "  return true_range\n",
    "\n",
    "\n",
    "def shift(array, place):\n",
    "\n",
    "  array = np.array(array, dtype=np.float32)\n",
    "  shifted = np.roll(array, place)\n",
    "  shifted[0:place] = np.nan\n",
    "  shifted[np.isnan(shifted)] = np.nanmean(shifted)\n",
    "\n",
    "  return shifted\n",
    "\n",
    "\n",
    "def ma_based_supertrend_indicator( high, low, close, atr_length=10, atr_multiplier=3, ma_length=10):\n",
    "\n",
    "    # Calculate True Range and Smoothed ATR\n",
    "    tr = true_range(high, low, close)\n",
    "    atr = ema(tr, atr_length)\n",
    "\n",
    "    upper_band = (high + low) / 2 + (atr_multiplier * atr)\n",
    "    lower_band = (high + low) / 2 - (atr_multiplier * atr)\n",
    "\n",
    "    trend = np.zeros(len(atr))\n",
    "\n",
    "    # Calculate Moving Average\n",
    "    ema_values = ema(close, ma_length)\n",
    "\n",
    "    if ema_values[0] > lower_band[0]:\n",
    "        trend[0] = lower_band[0]\n",
    "    elif ema_values[0] < upper_band[0]:\n",
    "        trend[0] = upper_band[0]\n",
    "    else:\n",
    "        trend[0] = upper_band[0]\n",
    "\n",
    "    # Compute final upper and lower bands\n",
    "    for i in range(1, len(close)):\n",
    "        if ema_values[i] > trend[i - 1]:\n",
    "            trend[i] = max(trend[i - 1], lower_band[i])\n",
    "\n",
    "\n",
    "        elif ema_values[i] < trend[i - 1]:\n",
    "            trend[i] = min(trend[i - 1], upper_band[i])\n",
    "\n",
    "        else:\n",
    "            trend[i] = trend[i - 1]\n",
    "\n",
    "    status_value = np.where(ema_values > trend, 1.0, -1.0)\n",
    "\n",
    "    return trend, status_value\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def supertrend_status_crossover( status_value):\n",
    "\n",
    "\n",
    "    prev_status = np.roll(status_value, 1)\n",
    "    supertrend_status_crossover = np.where((prev_status < 0) & (status_value > 0), 1.0, np.where((prev_status > 0) & (status_value < 0), -1.0, 0))\n",
    "\n",
    "    return supertrend_status_crossover\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def supertrend_indicator(high, low, close, period, multiplier=1.0):\n",
    "\n",
    "    true_range_value = true_range(high, low, close)\n",
    "\n",
    "    smoothed_atr = ema(true_range_value, period)\n",
    "\n",
    "    upper_band = (high + low) / 2 + (multiplier * smoothed_atr)\n",
    "    lower_band = (high + low) / 2 - (multiplier * smoothed_atr)\n",
    "\n",
    "    supertrend = np.zeros(len(true_range_value))\n",
    "    trend = np.zeros(len(true_range_value))\n",
    "\n",
    "    if close[0] > upper_band[0]: trend[0] = upper_band[0]\n",
    "    elif close[0] < lower_band[0]: trend[0] = lower_band[0]\n",
    "    else:  trend[0] = upper_band[0]\n",
    "\n",
    "    for i in range(1, len(close)):\n",
    "\n",
    "        if close[i] > upper_band[i]: trend[i] = upper_band[i]\n",
    "        elif close[i] < lower_band[i]: trend[i] = lower_band[i]\n",
    "        else: trend[i] = trend[i - 1]\n",
    "\n",
    "    # Calculate Buy/Sell Signals using numpy where  # np.where( close > trend, '1 Buy', '-1 Sell')\n",
    "    status_value = np.where(close > trend, 1.0, -1.0)\n",
    "\n",
    "    return trend, status_value\n",
    "\n",
    "def supertrend_status_crossover(status_value):\n",
    "\n",
    "\n",
    "    prev_status = np.roll(status_value, 1)\n",
    "    supertrend_status_crossover = np.where((prev_status < 0) & (status_value > 0), 1.0, np.where((prev_status > 0) & (status_value < 0), -1.0, 0))\n",
    "\n",
    "    return supertrend_status_crossover\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def direction_crossover_signal_line(signal, signal_ema):\n",
    "\n",
    "    direction = np.where(signal - signal_ema > 0, 1, -1)\n",
    "    prev_direction = np.roll(direction, 1)\n",
    "    crossover = np.where((prev_direction < 0) & (direction > 0), 1,\n",
    "                          np.where((prev_direction > 0) & (direction < 0), -1, 0))\n",
    "\n",
    "    return direction, crossover\n",
    "\n",
    "\n",
    "def stochastic_momentum_index(high, low, close, period=20, ema_period=5):\n",
    "    # Compute highest high and lowest low over the period\n",
    "    highest_high = high.rolling(window=period).max()\n",
    "    lowest_low = low.rolling(window=period).min()\n",
    "\n",
    "    # Compute relative range\n",
    "    relative_range = close - ((highest_high + lowest_low) / 2)\n",
    "\n",
    "    # Compute highest-lowest range\n",
    "    highest_lowest_range = highest_high - lowest_low\n",
    "\n",
    "    # Smooth relative range and highest-lowest range\n",
    "    relative_range_smoothed = relative_range.ewm(span=ema_period, adjust=False).mean().ewm(span=ema_period, adjust=False).mean()\n",
    "    highest_lowest_range_smoothed = highest_lowest_range.ewm(span=ema_period, adjust=False).mean().ewm(span=ema_period, adjust=False).mean()\n",
    "\n",
    "    # Calculate SMI\n",
    "    smi = (relative_range_smoothed / (highest_lowest_range_smoothed / 2)) * 100\n",
    "    smi[smi == np.inf] = 0  # Replace infinite values with 0\n",
    "    smi_ema = smi.ewm(span=ema_period, adjust=False).mean()\n",
    "\n",
    "    return smi, smi_ema\n",
    "\n",
    "# def stochastic_momentum_index(high, low, close, period=20, ema_period=5):\n",
    "\n",
    "#     lengthD = ema_period\n",
    "#     lowest_low   = moving_min(low, period)\n",
    "#     highest_high = moving_max(high, period)\n",
    "#     relative_range = close - ((highest_high + lowest_low) / 2)\n",
    "#     highest_lowest_range = highest_high - lowest_low\n",
    "\n",
    "#     relative_range_smoothed = ema(ema(relative_range, ema_period), ema_period)\n",
    "#     highest_lowest_range_smoothed = ema(ema(highest_lowest_range, ema_period), ema_period)\n",
    "\n",
    "#     smi = [(relative_range_smoothed[i] / (highest_lowest_range_smoothed[i] / 2)) * 100 if\n",
    "#             highest_lowest_range_smoothed[i] != 0 else 0.0\n",
    "#             for i in range(len(relative_range_smoothed))]\n",
    "\n",
    "#     smi_ema = ema(smi, ema_period)\n",
    "\n",
    "#     return smi, smi_ema\n",
    "\n",
    "\n",
    "def candle_type(o, h, l, c):\n",
    "\n",
    "    diff = abs(c - o)\n",
    "    o1, c1 = np.roll(o, 1), np.roll(c, 1)  #\n",
    "    min_oc = np.where(o < c, o, c)\n",
    "    max_oc = np.where(o > c, o, c)\n",
    "\n",
    "    pattern = np.where(\n",
    "      np.logical_and( min_oc - l > diff, h - max_oc < diff), 6,\n",
    "      np.where(np.logical_and( h - max_oc > diff, min_oc - l < diff),\n",
    "      4, np.where(np.logical_and(np.logical_and(c > o, c1 < o1), np.logical_and(c > o1, o < c1)),\n",
    "        5, np.where( min_oc - l > diff, 3,\n",
    "                      np.where(np.logical_and( h - max_oc > diff,\n",
    "                  min_oc - l < diff),\n",
    "                      2, np.where(np.logical_and(np.logical_and(c > o, c1 < o1), np.logical_and(c > o1, o < c1)),\n",
    "                      1, 0))))))\n",
    "    return pattern\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def heikin_ashi_status( ha_open, ha_close):\n",
    "\n",
    "    candles = np.full_like(ha_close, '', dtype='U10')\n",
    "\n",
    "    for i in range(1, len(ha_close)):\n",
    "\n",
    "        if ha_close[i] > ha_open[i]: candles[i] = 2 #'Green'\n",
    "\n",
    "        elif ha_close[i] < ha_open[i]: candles[i] = 1 # 'Red'\n",
    "\n",
    "        else: candles[i] = 0 #'Neutral'\n",
    "\n",
    "    return candles\n",
    "\n",
    "def heikin_ashi_candles( open, high, low, close):\n",
    "\n",
    "    ha_low, ha_close = np.empty(len(close), dtype=np.float32), np.empty(len(close), dtype=np.float32)\n",
    "    ha_open, ha_high = np.empty(len(close), dtype=np.float32), np.empty(len(close), dtype=np.float32)\n",
    "\n",
    "    ha_open[0] = (open[0] + close[0]) / 2\n",
    "    ha_close[0] = (close[0] + open[0] + high[0] + low[0]) / 4\n",
    "\n",
    "    for i in range(1, len(close)):\n",
    "        ha_open[i] = (ha_open[i - 1] + ha_close[i - 1]) / 2\n",
    "        ha_close[i] = (open[i] + high[i] + low[i] + close[i]) / 4\n",
    "        ha_high[i] = max(high[i], ha_open[i], ha_close[i])\n",
    "        ha_low[i] = min(low[i], ha_open[i], ha_close[i])\n",
    "\n",
    "    return ha_open, ha_close, ha_high, ha_low\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python",
   "language": "python",
   "name": "conda-env-python-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "toc-showcode": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 5
}