exp_player.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "575850bd-ac13-47f8-ac8f-781a6f9df921",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/env python3\n",
    "from Play.ENV import LuxuryDiceGame\n",
    "from Play.model import DiceMaster as Net\n",
    "from collections import deque, namedtuple, OrderedDict\n",
    "from tensorboardX import SummaryWriter\n",
    "from lib.configuration import Config\n",
    "from lib.utils import *\n",
    "\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2c359eb7-96a4-4601-b883-7b052238a5db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------- APP CONFIG ----------------------------------\n",
      "data: \n",
      "  data_path: gs://xxxx.csv\n",
      "  num_gaussian: 5\n",
      "  time_length: 16\n",
      "  train_test_split: 0.8\n",
      "train: \n",
      "  batch_size: 4\n",
      "  epoch: 50\n",
      "test: \n",
      "  batch_size: 2\n",
      "model: \n",
      "  loss: MeanSquaredError\n",
      "  metrics: MeanSquaredError\n",
      "optimizer: \n",
      "  method: adam\n",
      "  learning_rate: 1e-5\n",
      "weights: \n",
      "  simulation: weights/best_simulation_model.h5\n",
      "RL_PARAMETER: \n",
      "  MEAN_REWARD_BOUND: 19.5\n",
      "  INIT_BALANCE: 0\n",
      "  GAMMA: 0.99\n",
      "  BATCH_SIZE: 32\n",
      "  REPLAY_SIZE: 10000\n",
      "  OBSERVATION_SIZE: 15\n",
      "  LEARNING_RATE: 1e-4\n",
      "  SYNC_TARGET_FRAMES: 1000\n",
      "  REPLAY_START_SIZE: 10000\n",
      "  EPSILON_DECAY_LAST_EPOCH: 1e5\n",
      "  EPSILON_START: 1.0\n",
      "  EPSILON_FINAL: 0.02\n",
      "overflow: \n",
      "  size: 40000000\n",
      "  sequence: 20\n",
      "probabilityTable: [{'large': 4730, 'leopard': 510, 'small': 4760}, {'large': 4740, 'leopard': 480, 'small': 4780}, {'large': 4750, 'leopard': 450, 'small': 4800}, {'large': 4760, 'leopard': 430, 'small': 4810}]\n",
      "index_round: 100\n",
      "odds: \n",
      "  small: 2\n",
      "  leopard: 20\n",
      "  large: 2\n",
      "tax: \n",
      "  emergency: 0.01\n",
      "  loaded: 0.06\n",
      "  fair: 0.06\n",
      "safeline: 35000000\n",
      "loadedgate: 100000\n",
      "simulation_model_path: weights/best_luxury_dice_simulation_model.tf\n",
      "starting_data_path: gs://bin_for_aiops/GambleMaster/LuxuryDice/luxury_dice_v2.npz\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "config = Config(os.path.join(os.getcwd(), \"Play/config.yaml\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a5c6fb7-a73a-4064-99aa-93e99d2b6a23",
   "metadata": {},
   "source": [
    "### Set GPU as available physical device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6f77111d-1328-4a06-bd77-62ce272f6eee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-26 08:49:33.107542: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-12-26 08:49:33.120139: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-12-26 08:49:33.120426: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n"
     ]
    }
   ],
   "source": [
    "if gpus := tf.config.experimental.list_physical_devices(device_type='GPU'):\n",
    "    tf.config.experimental.set_visible_devices(devices=gpus[0], device_type='GPU')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "17fc7966-6a15-4157-ada3-aa8682cf2dbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_loss(batch):\n",
    "    states, actions, rewards, dones, next_states = batch\n",
    "    states_v = tf.constant(states)\n",
    "    next_states_v = tf.constant(next_states)\n",
    "    actions_v = tf.transpose(tf.constant(actions))\n",
    "    rewards_v = tf.transpose(tf.constant(rewards))\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        state_action_values = net(states_v)\n",
    "        next_state_values = tgt_net(next_states_v)\n",
    "        expected_state_action_values = next_state_values * config.RL_PARAMETER.GAMMA + rewards_v[:, tf.newaxis]\n",
    "        loss_t = tf.keras.losses.MSE(state_action_values, expected_state_action_values)\n",
    "    gradients = tape.gradient(loss_t, net.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, net.trainable_variables))\n",
    "    return loss_t.numpy()\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, env, exp_buffer):\n",
    "        self.env = env\n",
    "        self.exp_buffer = exp_buffer\n",
    "        self.state = None\n",
    "        self._reset()\n",
    "\n",
    "    def _reset(self):\n",
    "        self.state = self.env._reset()\n",
    "        self.total_reward = 0.0\n",
    "\n",
    "    def play_step(self, net, epsilon=0.0):\n",
    "        if np.random.random() < epsilon:\n",
    "            action = self.env.action_space.sample()\n",
    "        else:\n",
    "            state_a = np.array(list(self.env.states[\"Observation\"]) + list(self.env.states[\"TimeCode\"][0]), copy=False)\n",
    "            state_v = tf.constant(state_a.reshape(1, -1))\n",
    "            act_v = net(state_v)\n",
    "            action = {\"Bet\": tf.reduce_sum(act_v)*1e6, \"Category\": int(tf.argmax(act_v, axis=1))}\n",
    "\n",
    "        # do step in the environment\n",
    "        new_state, reward, is_done, call, prob = self.env.step(action)\n",
    "        self.total_reward += reward\n",
    "        exp = Experience(list(self.state[\"Observation\"]) + list(self.state[\"TimeCode\"].numpy()[0]),\n",
    "                         [float(action[\"Bet\"]), int(action[\"Category\"])],\n",
    "                         reward,\n",
    "                         is_done,\n",
    "                         list(new_state[\"Observation\"]) + list(new_state[\"TimeCode\"].numpy()[0]))\n",
    "        self.exp_buffer.append(exp)\n",
    "        self.state = new_state\n",
    "        if is_done:\n",
    "            self._reset()\n",
    "        return net, reward, action, call, prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f15d1a27-5a64-4655-b2e8-3c5586e2b9f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experience = namedtuple('Experience', field_names=['state', 'action', 'reward', 'done', 'new_state'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5df59ee9-7cb6-49c9-a3f1-330046cbbcb7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "Round: 10000, LastAverageLoss: 30113679360.000, LastAverageReward: -3021.415, CurrReward: 7866, eps: 0.90, Prob: {'large': 4750, 'leopard': 450, 'small': 4800}, Call: leopard, Bet: 414.10992431640625, Select: leopard\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-26 09:01:35.277231: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: weights/best_LuxuaryDiceModel.tf/assets\n"
     ]
    }
   ],
   "source": [
    "env = LuxuryDiceGame(config)\n",
    "net = Net()\n",
    "tgt_net = Net()\n",
    "\n",
    "buffer = ExperienceBuffer(config.RL_PARAMETER.REPLAY_SIZE)\n",
    "agent = Agent(env, buffer)\n",
    "epsilon = config.RL_PARAMETER.EPSILON_START\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=float(config.optimizer.learning_rate))\n",
    "round_idx = 0\n",
    "total_reward = 0\n",
    "best_mean_reward = None\n",
    "\n",
    "loss_cache = deque([], maxlen=3000)\n",
    "reward_cache = deque([], maxlen=3000)\n",
    "while True:\n",
    "    round_idx += 1\n",
    "    epsilon = max(config.RL_PARAMETER.EPSILON_FINAL, config.RL_PARAMETER.EPSILON_START - round_idx \\\n",
    "                  / float(config.RL_PARAMETER.EPSILON_DECAY_LAST_EPOCH))\n",
    "\n",
    "    net, reward, action, call, prob = agent.play_step(net, epsilon)\n",
    "    reward_cache.append(reward)\n",
    "    if len(buffer) < config.RL_PARAMETER.REPLAY_START_SIZE:\n",
    "        continue\n",
    "\n",
    "    batch = buffer.sample(config.RL_PARAMETER.BATCH_SIZE)\n",
    "    loss_cache.append(np.mean(calc_loss(batch)))\n",
    "\n",
    "    if round_idx % 500 == 0:\n",
    "        if isinstance(action, OrderedDict):\n",
    "            bet = action[\"Bet\"][0]\n",
    "        else:\n",
    "            bet = int(action[\"Bet\"])\n",
    "        select = selection_map[action[\"Category\"]]\n",
    "        print(f\"Round: {round_idx}, LastAverageLoss: {sum(loss_cache)/len(loss_cache):.3f}, LastAverageReward: {sum(reward_cache)/len(reward_cache):.3f}, CurrReward: {reward}, eps: {epsilon:.2f}, Prob: {prob}, Call: {call}, Bet: {bet}, Select: {select}\")\n",
    "\n",
    "    if total_reward is not None:\n",
    "        if best_mean_reward is None or best_mean_reward < total_reward:\n",
    "            net.save(f\"weights/best_LuxuaryDiceModel.tf\", save_format='tf')\n",
    "            if best_mean_reward is not None:\n",
    "                print(f\"Best mean reward updated {best_mean_reward:.3f} -> {total_reward:.3f}, model saved\")\n",
    "            best_mean_reward = total_reward\n",
    "\n",
    "    if round_idx % config.RL_PARAMETER.SYNC_TARGET_FRAMES == 0:\n",
    "        # tgt_net = tf.keras.models.clone_model(net)\n",
    "        tgt_net.set_weights(net.get_weights())\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b3083a5-5779-45fc-ad43-ff2a813f9261",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-6.m114",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-6:m114"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}