DDPG_CNN_Forward_Step_Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb475515",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime, timedelta\n",
    "import tensorflow as tf\n",
    "import tensorlayer as tl\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "# import our model and env\n",
    "from env import PortfolioEnv\n",
    "from ddpg import actor_critic ,replay_buffer, actor_critic_ts_cs\n",
    "from tensorflow.python.client import device_lib\n",
    "tf.config.list_physical_devices('GPU')\n",
    "#pd.show_versions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48812bfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare history \n",
    "\n",
    "stock1 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/嘉化能源.csv', header=0, index_col = 0, parse_dates = True)\n",
    "stock2 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/氯碱化工.csv', header=0, index_col = 0, parse_dates = True)\n",
    "stock3 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/中国平安.csv', header=0, index_col = 0, parse_dates = True)\n",
    "stock4 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/伊利股份.csv', header=0, index_col = 0, parse_dates = True)\n",
    "stock5 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/贵州茅台.csv', header=0, index_col = 0, parse_dates = True)\n",
    "stock6 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/陕鼓动力.csv', header=0, index_col = 0, parse_dates = True)\n",
    "stock7 = pd.read_csv('C:/Users/lee/Desktop/Trading_Result/Data/双汇发展.csv', header=0, index_col = 0, parse_dates = True)\n",
    "\n",
    "\n",
    "df = pd.concat([stock1, stock2, stock3, stock4, stock5, stock6, stock7], axis=1)\n",
    "df.columns = pd.MultiIndex.from_product([[\"s1\", \"s2\", \"s3\",'s4','s5','s6','s7'],['open', 'high', 'low', 'close', \"volume\"]], \n",
    "                                        names=['stock', 'price'])\n",
    "df = df.dropna()\n",
    "\n",
    "start = datetime.strptime(\"2013-10-22\", '%Y-%m-%d' )\n",
    "df_ = df.loc[start:]\n",
    "\n",
    "# set the env input\n",
    "history = df_ # the stock data\n",
    "abbreviation = [\"s1\", \"s2\", \"s3\",'s4','s5','s6','s7'] # name of stock\n",
    "steps = 200 # step for one eps\n",
    "trading_cost = 0.0025\n",
    "time_cost=0.00 # cost of holding equity\n",
    "window_length = 10 # for obs\n",
    "eps_move = 5 # move the start date after each rest\n",
    "sample_start_date = \"2016-05-01\"\n",
    "\n",
    "# build the replay_buffer\n",
    "buffer_size = 100 # the max size of buffer\n",
    "\n",
    "# build the actor, critic, and target network \n",
    "gamma = 0.99986 # discount factor\n",
    "state_shape = [None,len(abbreviation),window_length,5] # (none, m_stock, history_window, feature)\n",
    "action_shape = [None,len(abbreviation)] # (none, m_stock)\n",
    "a_learning_rate = 0.01 # learning rate for actor\n",
    "target_lr = 0.01 # learning rate for target, for stable\n",
    "c_learning_rate = 0.01 # learning rate for critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39dc31f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yfinance as yf\n",
    "\n",
    "stocks =  ['amt', 'axp', 'ba', 'cvx', 'jnj', 'ko', 'mcd', 'msft', 't', 'wmt']\n",
    "start = datetime(2012,1,1)\n",
    "end = datetime(2022,11,11)\n",
    "data = yf.download(stocks, start=start, end=end)\n",
    "data_close = data['Adj Close']\n",
    "data_close.plot()\n",
    "\n",
    "\n",
    "# using US stock\n",
    "\n",
    "\n",
    "df = data.drop('Close', axis=1)\n",
    "df = df[['Open','High','Low','Adj Close','Volume']]\n",
    "df.columns = df.columns.swaplevel(0, 1)\n",
    "df.sort_index(axis=1, level=0, inplace=True)\n",
    "df.columns = pd.MultiIndex.from_product([stocks,['open', 'high', 'low', 'close', \"volume\"]], \n",
    "                                        names=['stock', 'price'])\n",
    "df = df.dropna()\n",
    "\n",
    "start = datetime.strptime(\"2012-04-01\", '%Y-%m-%d' )\n",
    "df_ = df.loc[start:]\n",
    "\n",
    "# set the env input\n",
    "history = df_ # the stock data\n",
    "abbreviation = stocks # name of stock\n",
    "steps = 200 # step for one eps\n",
    "trading_cost = 0.0025\n",
    "time_cost=0.00 # cost of holding equity\n",
    "window_length = 10 # for obs\n",
    "eps_move = 0 # move the start date after each rest\n",
    "sample_start_date = \"2015-05-01\"\n",
    "\n",
    "# build the replay_buffer\n",
    "buffer_size = int(steps * 5) # the max size of buffer\n",
    "\n",
    "# build the actor, critic, and target network \n",
    "gamma = 1 # discount factor\n",
    "state_shape = [None,len(abbreviation),window_length,5] # (none, m_stock, history_window, feature)\n",
    "action_shape = [None,len(abbreviation)] # (none, m_stock)\n",
    "a_learning_rate = 0.01 # learning rate for actor\n",
    "target_lr = 0.001 # learning rate for target, for stable\n",
    "c_learning_rate = 0.001 # learning rate for critic\n",
    "\n",
    "alpha = 0\n",
    "beta = 0\n",
    "gamma_ = 0.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4728ba1",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_close['2016-02-01':'2017-04-01'].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55b3b346",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "RANDOM_SEED = 1234\n",
    "\n",
    "# build the env\n",
    "#PM_env = PM_env = PortfolioEnv.PortfolioEnv(history, abbreviation, steps, trading_cost, \\\n",
    "                                   #time_cost, window_length, eps_move, sample_start_date,\\\n",
    "                                   #alpha, beta, gamma_\n",
    "                                  #)\n",
    "# build the replay buffer\n",
    "#rb = replay_buffer.ReplayBuffer(buffer_size, RANDOM_SEED)\n",
    "\n",
    "# PM_env.seed(RANDOM_SEED)\n",
    "np.random.seed(RANDOM_SEED)\n",
    "tf.random.set_seed(RANDOM_SEED)\n",
    "\n",
    "state_dim = PM_env.observation_space.shape\n",
    "action_dim = PM_env.action_space.shape\n",
    "action_range = PM_env.action_space.high  # scale action, [-action_range, action_range]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc5df4f6",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e270960b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "\n",
    "# build out agents\n",
    "#  gamma, state_shape, action_shape, a_learning_rate, target_lr, c_learning_rate\n",
    "ac = actor_critic.Actor_Critic_NetWork(gamma, state_shape,action_shape,a_learning_rate,target_lr,c_learning_rate)\n",
    "\n",
    "\n",
    "TRAIN_EPISODES = 50\n",
    "\n",
    "batch_size = int(buffer_size/4) # mini batch\n",
    "\n",
    "train = True\n",
    "RENDER = False\n",
    "\n",
    "ALG_NAME = \"DDPG\"\n",
    "ENV_ID = \"Portfolio_Env\"\n",
    "\n",
    "\n",
    "\n",
    "t0 = time.time()\n",
    "if train:  # train\n",
    "    all_episode_reward = []\n",
    "    for episode in range(TRAIN_EPISODES):\n",
    "        state, info = PM_env.reset() \n",
    "        episode_reward = 0\n",
    "        for step in range(steps):\n",
    "            #print(state)\n",
    "            if RENDER:\n",
    "                PM_env.render()\n",
    "            # Add exploration noise\n",
    "            action = ac.Generate_action(state, greedy = False) #[[a0, a1, a2]]\n",
    "            #print(action)\n",
    "            state_, reward, done, info = PM_env.step(action) # compute the immidate reward and move to next steps\n",
    "            #print(info)\n",
    "            rb.add(state, action, reward, state_) # replay buffer add memory\n",
    "\n",
    "            if rb.count > buffer_size:\n",
    "                inputs = rb.sample_batch(batch_size)\n",
    "                ac.learn(inputs)\n",
    "\n",
    "            state = state_\n",
    "            episode_reward += reward * gamma**step\n",
    "            if done:\n",
    "                break\n",
    "\n",
    "        if episode == 0:\n",
    "            all_episode_reward.append(episode_reward)\n",
    "        else:\n",
    "            all_episode_reward.append(all_episode_reward[-1] * 0.9 + episode_reward * 0.1)\n",
    "        print(\n",
    "            'Training  | Episode: {}/{}  | Episode Reward: {:.4f}  | Running Time: {:.4f}'.format(\n",
    "                episode + 1, TRAIN_EPISODES, episode_reward,\n",
    "                time.time() - t0\n",
    "            )\n",
    "        )\n",
    "    ac.save()\n",
    "    plt.plot(all_episode_reward)\n",
    "    if not os.path.exists('image'):\n",
    "        os.makedirs('image')\n",
    "    plt.savefig(os.path.join('image', '_'.join([ALG_NAME, ENV_ID])))\n",
    "\n",
    "# Test our model\n",
    "\n",
    "test = True\n",
    "TEST_EPISODES = 1\n",
    "TEST_steps = steps\n",
    "\n",
    "\n",
    "# right after the end of train data\n",
    "#test_start_date = str(history.loc[sample_start_date:].index[steps])\n",
    "#test_start_date= test_start_date[0:10] # only need year-month-day\n",
    "test_start_date = sample_start_date\n",
    "\n",
    "\n",
    "PM_env = PortfolioEnv.PortfolioEnv(history, abbreviation, steps, trading_cost, \\\n",
    "                                   time_cost, window_length, eps_move, sample_start_date,\\\n",
    "                                   alpha, beta, gamma_\n",
    "                                  )\n",
    "\n",
    "weight_track = {}\n",
    "for name in abbreviation:\n",
    "    weight_track[name] = []\n",
    "    \n",
    "\n",
    "if test:\n",
    "    # test\n",
    "    ac.load() # load previous parameters\n",
    "    win_case = [] #used to count when cnn_model beat the equal weight portfolio\n",
    "    for episode in range(TEST_EPISODES):\n",
    "        state, info = PM_env.reset()\n",
    "        episode_reward = 0\n",
    "        for step in range(TEST_steps):\n",
    "            action = ac.Generate_action(state, greedy = True)\n",
    "            print(max(action[0]))\n",
    "            state, reward, done, info = PM_env.step(action)\n",
    "            #print(info)\n",
    "            for i in range(len(abbreviation)):\n",
    "                name = abbreviation[i]\n",
    "                weight_track[name].append(action[0][i])\n",
    "            episode_reward += reward * gamma**step\n",
    "            if done:\n",
    "                break\n",
    "        if PM_env.infos[-1][\"portfolio_value\"] > PM_env.infos[-1][\"equal_weight_portfolio_value\"]:\n",
    "            win_case.append(1) # if the CNN portfolio win\n",
    "        else:\n",
    "            win_case.append(0)\n",
    "        PM_env.render()\n",
    "        print(\n",
    "            'Testing  | Episode: {}/{}  | Episode Reward: {:.4f}  | Running Time: {:.4f}'.format(\n",
    "                episode + 1, TEST_EPISODES, episode_reward,\n",
    "                time.time() - t0\n",
    "            )\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52847aeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the weight plot\n",
    "\n",
    "\n",
    "\n",
    "x = np.arange(0,steps,1)\n",
    "for name in abbreviation:\n",
    "    y = weight_track[name]\n",
    "    plt.plot(x, y, label = name, linestyle=\"-.\")   \n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "179518ea",
   "metadata": {},
   "source": [
    "# Forward Stepping Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2737ed9",
   "metadata": {},
   "outputs": [],
   "source": [
    "eps_move = 5\n",
    "\n",
    "\n",
    "ac = actor_critic.Actor_Critic_NetWork(gamma, state_shape,action_shape,a_learning_rate,target_lr,c_learning_rate)\n",
    "warm_up = 15 # first 15 eps is for warm up only, test will perform after the warm up process\n",
    "warm_up_count = 0\n",
    "\n",
    "TRAIN_EPISODES = 65\n",
    "\n",
    "batch_size = int(buffer_size/10) # mini batch\n",
    "\n",
    "train = True\n",
    "RENDER = False\n",
    "\n",
    "ALG_NAME = \"DDPG\"\n",
    "ENV_ID = \"Portfolio_Env\"\n",
    "\n",
    "# trainning env\n",
    "PM_env = PortfolioEnv.PortfolioEnv(history, abbreviation, steps, trading_cost, \\\n",
    "                                   time_cost, window_length, eps_move, sample_start_date,\\\n",
    "                                   alpha, beta, gamma_\n",
    "                                  )\n",
    "rb = replay_buffer.ReplayBuffer(buffer_size, RANDOM_SEED)\n",
    "\n",
    "test = True\n",
    "TEST_EPISODES = TRAIN_EPISODES - warm_up\n",
    "TEST_step = eps_move # after the model is trained, n steps to test the data\n",
    "TEST_steps = TEST_step * TEST_EPISODES # total steps for test period\n",
    "\n",
    "\n",
    "# right after the end of train data\n",
    "test_start_date = str(history.loc[sample_start_date:].index[int(steps + warm_up * eps_move)])\n",
    "test_start_date= test_start_date[0:10] # only need year-month-day\n",
    "\n",
    "# test env\n",
    "test_PM_env = PortfolioEnv.PortfolioEnv(history, abbreviation, \\\n",
    "                                        TEST_steps, trading_cost, time_cost, \\\n",
    "                                        window_length, eps_move, test_start_date, \\\n",
    "                                        alpha, beta, gamma_\n",
    "                                       )\n",
    "\n",
    "weight_track = {}\n",
    "bp_weight_track = {}\n",
    "weighted_rank = []\n",
    "for name in abbreviation:\n",
    "    weight_track[name] = []\n",
    "    bp_weight_track[name] = []\n",
    "    \n",
    "\n",
    "t0 = time.time()\n",
    "if train:  # train\n",
    "    test_state, test_info = test_PM_env.reset()\n",
    "    for episode in range(TRAIN_EPISODES):\n",
    "        ###################################### Train the model ############################################\n",
    "        state, info = PM_env.reset()\n",
    "        episode_reward = 0\n",
    "        for step in range(steps):\n",
    "            if RENDER:\n",
    "                PM_env.render()\n",
    "            # Add exploration noise\n",
    "            action = ac.Generate_action(state, greedy = False) #[[a0, a1, a2]]\n",
    "            state_, reward, done, info = PM_env.step(action)\n",
    "            rb.add(state, action, reward, state_)\n",
    "\n",
    "            if rb.count > buffer_size:\n",
    "                inputs = rb.sample_batch(batch_size)\n",
    "                ac.learn(inputs)\n",
    "\n",
    "            state = state_\n",
    "            if done:\n",
    "                ac.save()\n",
    "                warm_up_count += 1\n",
    "                break\n",
    "        ###################################### Test the model ############################################\n",
    "        # test\n",
    "        if (warm_up_count > warm_up):\n",
    "            ac.load() # load previous parameters\n",
    "            for step in range(TEST_step):\n",
    "                test_action = ac.Generate_action(test_state, greedy = True)\n",
    "                test_state, reward, done, info = test_PM_env.step(test_action)\n",
    "                for i in range(len(abbreviation)):\n",
    "                    name = abbreviation[i]\n",
    "                    weight_track[name].append(test_action[0][i])\n",
    "                    bp_weight_track[name].append(test_PM_env.sim.bp_weight[i])\n",
    "                weighted_rank.append(test_PM_env.sim.weighted_rank)\n",
    "                if done:\n",
    "                    break\n",
    "                episode_reward += reward * gamma**step\n",
    "            print(\n",
    "                'Testing  | Episode: {}/{} | Episode Reward: {:.4f}   | Running Time: {:.4f}'.format(\n",
    "                    episode + 1, TRAIN_EPISODES, \n",
    "                    episode_reward,\n",
    "                    time.time() - t0\n",
    "                )\n",
    "            )\n",
    "        else:\n",
    "            print(f'Warm up process: Episode {episode + 1}')\n",
    "    test_PM_env.render()       \n",
    "    ac.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0feebc6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the weight plot, show the dynamic of portfolio\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "x = test_PM_env.date_track[0:TEST_steps]\n",
    "for name in abbreviation:\n",
    "    y = weight_track[name]\n",
    "    plt.plot(x, y, label = name, linestyle=\"-.\")\n",
    "\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Weight\")\n",
    "plt.legend(loc = 'upper right')\n",
    "plt.title(\"DDPG Dynamic weight\")\n",
    "plt.show()\n",
    "\n",
    "plt.plot(x, weighted_rank,  linestyle=\"-\")\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Weighted Rank\")\n",
    "y = len(abbreviation) * (len(abbreviation) - 1)/ (2 * len(abbreviation))\n",
    "plt.axhline(y = y, color = 'r', linestyle = '-')\n",
    "#plt.legend(loc = 'upper right')\n",
    "plt.title(\"Weighted Rank \")\n",
    "plt.show()\n",
    "\n",
    "\n",
    "for name in abbreviation:\n",
    "    z = bp_weight_track[name]\n",
    "    plt.plot(x, z, label = name, linestyle=\"-.\")\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Weight\")\n",
    "\n",
    "plt.legend(loc = 'upper right')\n",
    "plt.title(\"Best past stock \")\n",
    "plt.show()   \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e67e1e63",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_PM_env.table()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4182bffc",
   "metadata": {},
   "source": [
    "# Forward Stepping Test with CS analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c9dfe52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward Stepping Test\n",
    "\n",
    "eps_move = 5\n",
    "\n",
    "\n",
    "ac = actor_critic_ts_cs.Actor_Critic_NetWork(gamma, state_shape,action_shape,a_learning_rate,target_lr,c_learning_rate)\n",
    "warm_up = 15 # first 15 eps is for warm up only\n",
    "\n",
    "\n",
    "warm_up_count = 0\n",
    "\n",
    "TRAIN_EPISODES = 65\n",
    "\n",
    "\n",
    "batch_size = int(buffer_size/10) # mini batch\n",
    "\n",
    "train = True\n",
    "RENDER = False\n",
    "\n",
    "ALG_NAME = \"DDPG\"\n",
    "ENV_ID = \"Portfolio_Env\"\n",
    "\n",
    "# trainning env\n",
    "PM_env = PortfolioEnv.PortfolioEnv(history, abbreviation, steps, trading_cost, \\\n",
    "                                   time_cost, window_length, eps_move, sample_start_date,\\\n",
    "                                   alpha, beta, gamma_\n",
    "                                  )\n",
    "rb = replay_buffer.ReplayBuffer(buffer_size, RANDOM_SEED)\n",
    "\n",
    "test = True\n",
    "TEST_EPISODES = TRAIN_EPISODES - warm_up\n",
    "TEST_step = eps_move # after the model is trained, n steps to test the data\n",
    "TEST_steps = TEST_step * TEST_EPISODES # total steps for test period\n",
    "\n",
    "\n",
    "# right after the end of train data\n",
    "test_start_date = str(history.loc[sample_start_date:].index[int(steps + warm_up * eps_move)])\n",
    "test_start_date= test_start_date[0:10] # only need year-month-day\n",
    "\n",
    "\n",
    "# test env\n",
    "test_PM_env = PortfolioEnv.PortfolioEnv(history, abbreviation, \\\n",
    "                                        TEST_steps, trading_cost, time_cost, \\\n",
    "                                        window_length, eps_move, test_start_date, \\\n",
    "                                        alpha, beta, gamma_\n",
    "                                       )\n",
    "\n",
    "weight_track = {}\n",
    "bp_weight_track = {}\n",
    "weighted_rank = []\n",
    "for name in abbreviation:\n",
    "    weight_track[name] = []\n",
    "    bp_weight_track[name] = []\n",
    "    \n",
    "\n",
    "t0 = time.time()\n",
    "if train:  # train\n",
    "    test_state, test_info = test_PM_env.reset()\n",
    "    for episode in range(TRAIN_EPISODES):\n",
    "        ###################################### Train the model ############################################\n",
    "        state, info = PM_env.reset()\n",
    "        episode_reward = 0\n",
    "        for step in range(steps):\n",
    "            if RENDER:\n",
    "                PM_env.render()\n",
    "            # Add exploration noise\n",
    "            action = ac.Generate_action(state, greedy = False) #[[a0, a1, a2]]\n",
    "            state_, reward, done, info = PM_env.step(action)\n",
    "            rb.add(state, action, reward, state_)\n",
    "\n",
    "            if rb.count > buffer_size:\n",
    "                inputs = rb.sample_batch(batch_size)\n",
    "                ac.learn(inputs)\n",
    "\n",
    "            state = state_\n",
    "            if done:\n",
    "                ac.save()\n",
    "                warm_up_count += 1\n",
    "                break\n",
    "        ###################################### Test the model ############################################\n",
    "        # test\n",
    "        if (warm_up_count > warm_up):\n",
    "            ac.load() # load previous parameters\n",
    "            for step in range(TEST_step):\n",
    "                test_action = ac.Generate_action(test_state, greedy = True)\n",
    "                test_state, reward, done, info = test_PM_env.step(test_action)\n",
    "                for i in range(len(abbreviation)):\n",
    "                    name = abbreviation[i]\n",
    "                    weight_track[name].append(test_action[0][i])\n",
    "                    bp_weight_track[name].append(test_PM_env.sim.bp_weight[i])\n",
    "                weighted_rank.append(test_PM_env.sim.weighted_rank)\n",
    "                if done:\n",
    "                    break\n",
    "                episode_reward += reward * gamma**step\n",
    "            print(\n",
    "                'Testing  | Episode: {}/{} | Episode Reward: {:.4f}   | Running Time: {:.4f}'.format(\n",
    "                    episode + 1, TRAIN_EPISODES, \n",
    "                    episode_reward,\n",
    "                    time.time() - t0\n",
    "                )\n",
    "            )\n",
    "        else:\n",
    "            print(f'Warm up process: Episode {episode + 1}')\n",
    "    test_PM_env.render()       \n",
    "    ac.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b4169a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the weight plot, show the dynamic of portfolio\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "x = test_PM_env.date_track[0:TEST_steps]\n",
    "for name in abbreviation:\n",
    "    y = weight_track[name]\n",
    "    plt.plot(x, y, label = name, linestyle=\"-.\")\n",
    "    \n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Weight\")\n",
    "plt.legend(loc = 'upper right')\n",
    "plt.title(\"DDPG_CS Dynamic weight\")\n",
    "plt.show()\n",
    "\n",
    "plt.plot(x, weighted_rank,  linestyle=\"-\")\n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Weighted Rank\")\n",
    "y = len(abbreviation) * (len(abbreviation) - 1)/ (2 * len(abbreviation))\n",
    "plt.axhline(y = y, color = 'r', linestyle = '-')\n",
    "#plt.legend(loc = 'upper right')\n",
    "plt.title(\"Weighted Rank \")\n",
    "plt.show()\n",
    "\n",
    "\n",
    "for name in abbreviation:\n",
    "    z = bp_weight_track[name]\n",
    "    plt.plot(x, z, label = name, linestyle=\"-.\")\n",
    "    \n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Weight\")\n",
    "\n",
    "plt.legend(loc = 'upper right')\n",
    "plt.title(\"Best past stock \")\n",
    "plt.show()   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5630c7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_PM_env.table()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}