examples/Notebooks/tutorials/Collinear_GS_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Space Navigator with Collinear Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from space_navigator.models.CE import CrossEntropy\n",
    "from space_navigator.models.collinear_GS import CollinearGridSearch\n",
    "from space_navigator.utils import read_environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_path = \"../../../data/environments/collision.env\"\n",
    "env = read_environment(env_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "        Agrs:\n",
      "            env (Environment): environment with given parameteres.\n",
      "            step (float): time step in simulation.\n",
      "            reverse (bool): \n",
      "                if True: there are selected exactly 2 maneuvers\n",
      "                    while the second of them is reversed to the first one;\n",
      "                if False: one maneuver.\n",
      "            first_maneuver_direction (str): first maneuver is collinear\n",
      "                to the velocity vector and could be:\n",
      "                    \"forward\" (co-directed)\n",
      "                    \"backward\" (oppositely directed)\n",
      "                    \"auto\" (just collinear).\n",
      "        \n"
     ]
    }
   ],
   "source": [
    "# __init__ docstring\n",
    "print(CollinearGridSearch.__init__.__doc__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_parameters = {\n",
    "    \"env\": env,\n",
    "    \"step\": 1e-6,\n",
    "    \"reverse\": True, \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "collinear_GS_model = CollinearGridSearch(**init_parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Iteration\n",
    "\n",
    "(currently it is just one iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training iteration.\n",
      "\n",
      "        Args:\n",
      "            print_out (bool): print iteration information.\n",
      "            n_sessions (int): number of sessions to generate.\n",
      "\n",
      "        Returns:\n",
      "            stop (bool): whether to stop training after iteration.\n",
      "\n",
      "        \n"
     ]
    }
   ],
   "source": [
    "# iteration docstring\n",
    "print(collinear_GS_model.iteration.__doc__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "iteration_parameters = {\n",
    "    \"n_sessions\": 1000,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one iteration\n",
    "# collinear_GS_model.iteration(**iteration_parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training agent policy (self.action_table).\n",
      "\n",
      "        Args:\n",
      "            n_iterations (int): number of iterations.\n",
      "            print_out (bool): print information during the training.\n",
      "            *args and **kwargs: iteration arguments, depend on method (inheritor class).\n",
      "\n",
      "        TODO:\n",
      "            add early stopping\n",
      "            add log\n",
      "            decorate by print_out and log?\n",
      "        \n"
     ]
    }
   ],
   "source": [
    "# train docstring\n",
    "print(collinear_GS_model.train.__doc__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iterations = 1\n",
    "print_out = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 11/1000 [00:00<00:09, 107.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Start training.\n",
      "\n",
      "Initial action table:\n",
      "[]\n",
      "Initial Reward: -13489.530475182863\n",
      "\n",
      "iteration: 1/1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:09<00:00, 105.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training completed in 9.4744 sec.\n",
      "Total Reward: -0.5260470455285451\n",
      "Action Table:\n",
      "[[ 0.          0.          0.          0.        ]\n",
      " [ 1.60913022  1.51055235  0.02636679  0.07934858]\n",
      " [-1.60913022 -1.51055235 -0.02636679         nan]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "collinear_GS_model.train(n_iterations, print_out, **iteration_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 1.60913022,  1.51055235,  0.02636679,  0.07934858],\n",
       "       [-1.60913022, -1.51055235, -0.02636679,         nan]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# obtained table of actions\n",
    "collinear_GS_model.action_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_path =\n",
    "# collinear_GS_model.save_action_table(save_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "CE_init_parameters = {\n",
    "    \"env\": env,\n",
    "    \"step\": 1e-6,\n",
    "    \"reverse\": True, \n",
    "    \"first_maneuver_time\": 'early',\n",
    "    \"n_maneuvers\": 2,\n",
    "    \"lr\": 0.9,\n",
    "    \"percentile\": 95,\n",
    "    \"sigma_dV\": None,\n",
    "    \"sigma_t\": None,\n",
    "}\n",
    "CE_iteration_parameters = {\n",
    "    \"n_sessions\": 100,\n",
    "    \"sigma_decay\": 0.9,\n",
    "    \"lr_decay\": 0.9,\n",
    "    \"percentile_growth\": 1.01,\n",
    "    \"show_progress\": False,\n",
    "    \"dV_angle\": 'complanar',\n",
    "    \"step_if_low_reward\": False,\n",
    "    \"early_stopping\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tune_CE_model = CrossEntropy(**init_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 8/100 [00:00<00:01, 76.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Start training.\n",
      "\n",
      "Initial action table:\n",
      "[[ 0.          0.          0.          0.        ]\n",
      " [ 1.60913022  1.51055235  0.02636679  0.07934858]\n",
      " [-1.60913022 -1.51055235 -0.02636679         nan]]\n",
      "Initial Reward: -0.5260470455285451\n",
      "\n",
      "iteration: 1/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 91.56it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 95.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -233.1354991146465\n",
      "Max Reward:    -15.547391400031907\n",
      "Threshold:     -35.594369310229496\n",
      "\n",
      "iteration: 2/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.42it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 95.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -169.2179326244298\n",
      "Max Reward:    -0.6648954874118923\n",
      "Threshold:     -22.661941460580636\n",
      "\n",
      "iteration: 3/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 100.03it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 98.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -134.13024309572742\n",
      "Max Reward:    -4.505469222120893\n",
      "Threshold:     -14.713987741470781\n",
      "\n",
      "iteration: 4/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 95.37it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 96.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -135.9158082609062\n",
      "Max Reward:    -23.791370914330287\n",
      "Threshold:     -23.95303424854044\n",
      "\n",
      "iteration: 5/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.61it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 97.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -88.79543734816703\n",
      "Max Reward:    -9.397669260166847\n",
      "Threshold:     -13.71184888257834\n",
      "\n",
      "iteration: 6/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.45it/s]\n",
      " 11%|█         | 11/100 [00:00<00:00, 102.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -72.3236538347343\n",
      "Max Reward:    -3.3561204523590993\n",
      "Threshold:     -15.970342113724119\n",
      "\n",
      "iteration: 7/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 101.41it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 98.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -62.13333797135094\n",
      "Max Reward:    -1.851620578046373\n",
      "Threshold:     -3.669993854349833\n",
      "\n",
      "iteration: 8/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.83it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 98.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -56.060379718053845\n",
      "Max Reward:    -0.4292906125915815\n",
      "Threshold:     -5.535676195610343\n",
      "\n",
      "iteration: 9/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 92.66it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 91.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -52.1756787073549\n",
      "Max Reward:    -0.6909815705539876\n",
      "Threshold:     -5.451149769021014\n",
      "\n",
      "iteration: 10/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.51it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 95.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -53.17564993041132\n",
      "Max Reward:    -3.8372358305704988\n",
      "Threshold:     -6.523882662034369\n",
      "\n",
      "iteration: 11/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 95.29it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 95.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -43.94856124427077\n",
      "Max Reward:    -2.8244018009937752\n",
      "Threshold:     -6.125561430573137\n",
      "\n",
      "iteration: 12/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.40it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 95.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -48.10064367425386\n",
      "Max Reward:    -1.3834131768786952\n",
      "Threshold:     -5.833776932025648\n",
      "\n",
      "iteration: 13/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 100.22it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 99.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -49.768805839307475\n",
      "Max Reward:    -0.800628829073637\n",
      "Threshold:     -3.4054794690021706\n",
      "\n",
      "iteration: 14/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.10it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 94.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5260470455285451\n",
      "Mean Reward:   -50.59319638939412\n",
      "Max Reward:    -4.138963937982124\n",
      "Threshold:     -4.401670122848138\n",
      "\n",
      "iteration: 15/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 97.85it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 98.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5258205239009067\n",
      "Mean Reward:   -41.677995515114986\n",
      "Max Reward:    -0.6866365611725413\n",
      "Threshold:     -1.3160421641605686\n",
      "\n",
      "iteration: 16/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 99.88it/s]\n",
      " 10%|█         | 10/100 [00:00<00:00, 99.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5258205239009067\n",
      "Mean Reward:   -49.26818196859784\n",
      "Max Reward:    -0.819215519944822\n",
      "Threshold:     -1.5131960491771859\n",
      "\n",
      "iteration: 17/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 97.70it/s]\n",
      "  9%|▉         | 9/100 [00:00<00:01, 86.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5258205239009067\n",
      "Mean Reward:   -42.02128629126624\n",
      "Max Reward:    -0.7803658993882148\n",
      "Threshold:     -1.0315060850902806\n",
      "\n",
      "iteration: 18/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 95.03it/s]\n",
      " 11%|█         | 11/100 [00:00<00:00, 101.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5258205239009067\n",
      "Mean Reward:   -42.947304519446334\n",
      "Max Reward:    -0.5292041812633463\n",
      "Threshold:     -0.7133077016589888\n",
      "\n",
      "iteration: 19/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 101.25it/s]\n",
      " 11%|█         | 11/100 [00:00<00:00, 102.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5258205239009067\n",
      "Mean Reward:   -44.61188542629712\n",
      "Max Reward:    -0.6836780165430262\n",
      "Threshold:     -0.7947430344716759\n",
      "\n",
      "iteration: 20/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 101.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Policy Reward: -0.5258205239009067\n",
      "Mean Reward:   -45.424706776887234\n",
      "Max Reward:    -0.5288992337292051\n",
      "Threshold:     -0.67696453540506\n",
      "\n",
      "Early stopping.\n",
      "\n",
      "Training completed in 20.618 sec.\n",
      "Total Reward: -0.5258205239009067\n",
      "Action Table:\n",
      "[[ 0.          0.          0.          0.        ]\n",
      " [ 1.60131478  1.51420806  0.0264306   0.07934858]\n",
      " [-1.60131478 -1.51420806 -0.0264306          nan]]\n",
      "Actions table after tuning:\n",
      "[[ 0.          0.          0.          0.        ]\n",
      " [ 1.60131478  1.51420806  0.0264306   0.07934858]\n",
      " [-1.60131478 -1.51420806 -0.0264306          nan]]\n",
      "\n",
      "Reward: -0.5258205239009067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 50\n",
    "print_out = True\n",
    "\n",
    "# set initial actions table for tuning\n",
    "tune_CE_model.set_action_table(collinear_GS_model.action_table)\n",
    "# tuning\n",
    "tune_CE_model.train(n_iterations, print_out, **CE_iteration_parameters)\n",
    "\n",
    "print(f\"Actions table after tuning:\\n{tune_CE_model.action_table}\")\n",
    "print(f\"\\nReward: {tune_CE_model.policy_reward}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}