Merge pull request #126 from pockerman/feature_120_use_api_server_to_…

…make_requests

Feature 120 use api server to make requests

CMakeLists.txt

@@ -95,8 +95,8 @@ INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
 INCLUDE_DIRECTORIES(src/)
 
 FILE(GLOB SRCS src/rlenvs/*.cpp
-               src/rlenvs/api_server/*.cpp
 			   src/rlenvs/envs/*.cpp
+               src/rlenvs/envs/api_server/*.cpp
                src/rlenvs/envs/gymnasium/*.cpp
                src/rlenvs/envs/gymnasium/toy_text/*.cpp
                src/rlenvs/envs/gymnasium/classic_control/*.cpp

README.md

@@ -12,8 +12,8 @@ Currently, we provide a minimal number of wrappers for some common Gymnasium (fo
 | FrozenLake 8x8 map  |   Yes    | TODO                                                      |
 | Blackjack           |   Yes    | <a href="examples/example_1/example_1.cpp">example_1</a>  |
 | CliffWalking        |   Yes    | <a href="examples/example_1/example_1.cpp">example_1</a>  |
-| CartPole            |   Yes    | TODO                                                     |
-| MountainCar         |   Yes    | TODO                                                     |
+| CartPole            |   Yes    | TODO                                                      |
+| MountainCar         |   Yes    | TODO                                                      |
 | Taxi                |   Yes    | <a href="examples/example_1/example_1.cpp">example_1</a>  |
 | Pendulum            |   Yes    | <a href="examples/example_6/example_6.cpp">example_6</a>  |
 | Acrobot             |   Yes    | TODO                                                      |
@@ -26,9 +26,9 @@ Currently, we provide a minimal number of wrappers for some common Gymnasium (fo
 
 There exist some wrappers for vector environments: 
 
-| Environment         |   REST   | Example |
-| :----------------   | :------: | :----:  |
-| AcrobotV            |   Yes    |  TODO   |
+| Environment         |   REST   | Example                                                    |
+| :----------------   | :------: | :----:                                                     |
+| AcrobotV            |   Yes    |  <a href="examples/example_8/example_8.cpp">example_8</a>  |
 
 Various RL algorithms using the environments can be found at <a href="https://github.com/pockerman/cuberl/tree/master">cuberl</a>
 

examples/example_1/example_1.cpp

@@ -3,6 +3,7 @@
 #include "rlenvs/envs/gymnasium/toy_text/taxi_env.h"
 #include "rlenvs/envs/gymnasium/toy_text/black_jack_env.h"
 #include "rlenvs/envs/gymnasium/toy_text/cliff_world_env.h"
+#include "rlenvs/envs/api_server/apiserver.h"
 
 #include <iostream>
 #include <string>
@@ -13,9 +14,16 @@ namespace example_1{
 
 const std::string SERVER_URL = "http://0.0.0.0:8001/api";
 
-void test_frozen_lake(){
+using rlenvscpp::envs::gymnasium::FrozenLake;
+using rlenvscpp::envs::gymnasium::Taxi;
+using rlenvscpp::envs::gymnasium::BlackJack;
+using rlenvscpp::envs::gymnasium::CliffWorld;
+using rlenvscpp::envs::RESTApiServerWrapper;
 
-    rlenvscpp::envs::gymnasium::FrozenLake<4> env(SERVER_URL);
+
+void test_frozen_lake(const RESTApiServerWrapper& server){
+
+    FrozenLake<4> env(server);
 
     std::cout<<"Environame URL: "<<env.get_url()<<std::endl;
 
@@ -81,9 +89,9 @@ void test_frozen_lake(){
 
 }
 
-void test_taxi(){
+void test_taxi(const RESTApiServerWrapper& server){
 
-    rlenvscpp::envs::gymnasium::Taxi env(SERVER_URL);
+    Taxi env(server);
 
     std::cout<<"Environame URL: "<<env.get_url()<<std::endl;
 
@@ -145,9 +153,9 @@ void test_taxi(){
 }
 
 
-void test_black_jack(){
+void test_black_jack(const RESTApiServerWrapper& server){
 
-    rlenvscpp::envs::gymnasium::BlackJack env(SERVER_URL);
+    BlackJack env(server);
     std::unordered_map<std::string, std::any> options;
     options["natural"] = true;
 
@@ -180,9 +188,9 @@ void test_black_jack(){
 }
 
 
-void test_cliff_world(){
+void test_cliff_world(const RESTApiServerWrapper& server){
 
-	rlenvscpp::envs::gymnasium::CliffWorld env(SERVER_URL);
+	CliffWorld env(server);
 
     std::cout<<"Environment URL: "<<env.get_url()<<std::endl;
 
@@ -247,18 +255,21 @@ void test_cliff_world(){
 
 int main(){
 
+	using namespace example_1;
+
+	RESTApiServerWrapper server(SERVER_URL, true);
 
     std::cout<<"Testing FrozenLake..."<<std::endl;
-    example_1::test_frozen_lake();
+    example_1::test_frozen_lake(server);
     std::cout<<"===================="<<std::endl;
     std::cout<<"Testing Taxi..."<<std::endl;
-    example_1::test_taxi();
+    example_1::test_taxi(server);
     std::cout<<"===================="<<std::endl;
     std::cout<<"Testing BlackJack..."<<std::endl;
-    example_1::test_black_jack();
+    example_1::test_black_jack(server);
     std::cout<<"===================="<<std::endl;
     std::cout<<"Testing CliffWorld..."<<std::endl;
-    example_1::test_cliff_world();
+    example_1::test_cliff_world(server);
     std::cout<<"===================="<<std::endl;
     return 0;
 }

examples/example_2/example_2.cpp

@@ -1,12 +1,12 @@
-#include "rlenvs/api_server/apiserver.h"
+#include "rlenvs/envs/api_server/apiserver.h"
 #include <iostream>
 #include <string>
 
 int main(){
 
     const std::string SERVER_URL = "http://0.0.0.0:8001/api";
 
-    rlenvscpp::ApiServerWrapper server_wrapper(SERVER_URL);
+    rlenvscpp::envs::RESTApiServerWrapper server_wrapper(SERVER_URL);
 
     auto has_gym = server_wrapper.has_gymnasium();
     std::cout<<"Has environment server Gymnasium? "<<has_gym<<std::endl;

examples/example_3/example_3.cpp

@@ -1,6 +1,7 @@
 
 #include "rlenvs/rlenvs_types_v2.h"
 #include "rlenvs/envs/gymnasium/toy_text/frozen_lake_env.h"
+#include "rlenvs/envs/api_server/apiserver.h"
 #include "rlenvs/envs/envs_utils.h"
 #include "rlenvs/rlenvscpp_config.h"
 
@@ -16,6 +17,7 @@
 namespace example{
 
 using rlenvscpp::uint_t;
+using rlenvscpp::envs::RESTApiServerWrapper;
 const std::string SERVER_URL = "http://0.0.0.0:8001/api";
 const uint_t MAX_TRAJECTORY_SIZE = 10;
 
@@ -38,7 +40,8 @@ int main(){
 
     using namespace example;
 
-    env_type env(SERVER_URL);
+	RESTApiServerWrapper server(SERVER_URL, true);
+    env_type env(server);
 
     std::cout<<"Environame URL: "<<env.get_url()<<std::endl;
 

examples/example_6/example_6.cpp

@@ -1,5 +1,6 @@
 
 #include "rlenvs/envs/gymnasium/classic_control/pendulum_env.h"
+#include "rlenvs/envs/api_server/apiserver.h"
 #include "rlenvs/rlenvs_types_v2.h"
 #include "rlenvs/rlenvs_consts.h"
 
@@ -19,10 +20,13 @@
 int main(){
 
 	using namespace rlenvscpp::envs::gymnasium;
+	using rlenvscpp::envs::RESTApiServerWrapper;
 
-	const std::string url = "http://0.0.0.0:8001/api";
+	const std::string SERVER_URL = "http://0.0.0.0:8001/api";
 
-	Pendulum env(url);
+	RESTApiServerWrapper server(SERVER_URL, true);
+
+	Pendulum env(server);
 
 	std::cout<<"Name: "<<env.name<<std::endl;
 	std::cout<<"Number of actions: "<<env.n_actions()<<std::endl;

examples/example_8/example_8.cpp

@@ -1,5 +1,6 @@
 #include "rlenvs/rlenvs_types_v2.h"
 #include "rlenvs/envs/gymnasium/classic_control/vector/acrobot_vec_env.h"
+#include "rlenvs/envs/api_server/apiserver.h"
 #include "rlenvs/rlenvs_consts.h"
 
 #include <iostream>
@@ -14,11 +15,14 @@ int main(){
 
 	using namespace rlenvscpp::envs::gymnasium;
 	using rlenvscpp::uint_t;
+	using rlenvscpp::envs::RESTApiServerWrapper;
 
-	const std::string url = "http://0.0.0.0:8001/api";
+	const std::string SERVER_URL = "http://0.0.0.0:8001/api";
 
+	RESTApiServerWrapper server(SERVER_URL, true);
+
 	// Acrobot vector environment
-	AcrobotV env(url);
+	AcrobotV env(server);
 
 	std::cout<<"Name: "<<env.name<<std::endl;
 	std::cout<<"Number of actions: "<<env.n_actions()<<std::endl;

rest_api/gdrl/gym_walk_env/gym_walk_env_api.py

@@ -59,11 +59,20 @@ async def close(cidx: int) -> JSONResponse:
 @gym_walk_env_router.post("/make")
 async def make(version: str = Body(default="v1"),
                cidx: int = Body(...),
-               n_states: int = Body(default=7, ge=2),
-               p_stay: float = Body(default=0.0, ge=0.0, le=1.0),
-               p_backward: float = Body(default=0.5, le=1.0, ge=0.0)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={"n_states": 7,
+                                                       "p_stay": 0.0,
+                                                       "p_backward": 0.5}),
+              ) -> JSONResponse:
     global envs
 
+    # n_states: int = Body(default=7, ge=2),
+    # p_stay: float = Body(default=0.0, ge=0.0, le=1.0),
+    # p_backward: float = Body(default=0.5, le=1.0, ge=0.0)
+
+    n_states = options.get("n_states", 7)
+    p_stay = options.get("p_stay", 0.0)
+    p_backward = options.get("p_backward", 0.5)
+
     if cidx in envs:
         env = envs[cidx]
 

rest_api/gymnasium_envs/classic_control/acrobot_env_api.py

@@ -66,7 +66,7 @@ async def close(cidx: int) -> JSONResponse:
 
 @acrobot_router.post("/make")
 async def make(version: str = Body(default="v1"), cidx: int = Body(...),
-               max_episode_steps: int = Body(default=500)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={})) -> JSONResponse:
     global envs
     env_type = f"{ENV_NAME}-{version}"
     if cidx in envs:

rest_api/gymnasium_envs/classic_control/cart_pole_env_api.py

@@ -61,9 +61,7 @@ async def close(cidx: int) -> JSONResponse:
 @cart_pole_router.post("/make")
 async def make(version: str = Body(default="v1"),
                cidx: int = Body(...),
-               natural: bool = Body(default=False),
-               sab: bool = Body(default=False),
-               max_episode_steps: int = Body(default=500)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={})) -> JSONResponse:
     global envs
     env_type = f"{ENV_NAME}-{version}"
     if cidx in envs:
@@ -73,7 +71,7 @@ async def make(version: str = Body(default="v1"),
             envs[cidx].close()
 
         try:
-            env = gym.make(env_type, natural, sab)
+            env = gym.make(env_type,)
             envs[cidx] = env
         except Exception as e:
             logger.error('An exception was raised')
@@ -82,7 +80,7 @@ async def make(version: str = Body(default="v1"),
                                 detail=str(e))
     else:
         try:
-            env = gym.make(env_type, natural, sab)
+            env = gym.make(env_type)
             envs[cidx] = env
         except Exception as e:
             logger.error('An exception was raised')

rest_api/gymnasium_envs/classic_control/mountain_car_env_api.py

@@ -63,10 +63,11 @@ async def close(cidx: int) -> JSONResponse:
 @mountain_car_router.post("/make")
 async def make(version: str = Body(default="v0"),
                cidx: int = Body(...),
-               max_episode_steps: int = Body(default=200)
+               options: dict[str, Any] = Body(default={"max_episode_steps": 200})
                ) -> JSONResponse:
     global envs
     env_type = f"{ENV_NAME}-{version}"
+    max_episode_steps = options.get("max_episode_steps", 200)
     if cidx in envs:
         env = envs[cidx]
 

rest_api/gymnasium_envs/classic_control/pendulum_env_api.py

@@ -70,10 +70,11 @@ async def close(cidx: int) -> JSONResponse:
 @pendulum_router.post("/make")
 async def make(version: str = Body(default="v1"),
                cidx: int = Body(...),
-               g: float = Body(default=10.0),
-               max_episode_steps: int = Body(default=200)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={"g": 10.0, "max_episode_steps": 200})) -> JSONResponse:
     global envs
     env_type = f"{ENV_NAME}-{version}"
+    g = options.get("g", 10.0)
+    max_episode_steps = options.get("max_episode_steps", 200)
     if cidx in envs:
         env = envs[cidx]
 

rest_api/gymnasium_envs/classic_control/v/acrobot_v_env_api.py

@@ -86,14 +86,13 @@ async def close(cidx: int) -> JSONResponse:
 @acrobot_v_router.post("/make")
 async def make(version: str = Body(default="v1"),
                cidx: int = Body(...),
-               num_envs: int = Body(default=2),
-               max_episode_steps: int = Body(default=500)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={"num_envs": 2})) -> JSONResponse:
     global env
     global NUM_COPIES
     global envs
 
     env_type = f"{ENV_NAME}-{version}"
-
+    num_envs = options.get("num_envs", 2)
     if cidx in envs:
         env = envs[cidx]
 
@@ -164,11 +163,11 @@ async def reset(seed: int = Body(default=42), cidx: int = Body(...),
 
 
 @acrobot_v_router.post("/step")
-async def step(actions: dict[str, list[int]] = Body(title='actions'), cidx: int = Body(...)) -> JSONResponse:
+async def step(action: dict[str, list[int]] = Body(title='actions'), cidx: int = Body(...)) -> JSONResponse:
 
     global NUM_COPIES
 
-    actions = actions['actions']
+    actions = action['actions']
     if len(actions) != NUM_COPIES:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
                             detail=f"actions length should be {NUM_COPIES} ")

rest_api/gymnasium_envs/toy_text/black_jack_env_api.py

@@ -57,10 +57,11 @@ async def close(cidx: int) -> JSONResponse:
 @black_jack_router.post("/make")
 async def make(version: str = Body(default="v1"),
                cidx: int = Body(...),
-               natural: bool = Body(default=False),
-               sab: bool = Body(default=False)):
+               options: dict[str, Any] = Body(default={"natural": False, "sab": False})) -> JSONResponse:
     global envs
 
+    natural = options.get("natural", False)
+    sab = options.get("sab", False)
     if cidx in envs:
         env = envs[cidx]
 

rest_api/gymnasium_envs/toy_text/frozenlake_env_api.py

@@ -53,19 +53,25 @@ async def close(cidx: int) -> JSONResponse:
 @frozenlake_router.post("/make")
 async def make(version: str = Body(default='v1'),
                cidx: int = Body(...),
-               map_name: str = Body(default="4x4"),
-               is_slippery: bool = Body(default=True), max_episode_steps: int = Body(default=500)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={"map_name": "4x4",
+                                                       "is_slippery": True,
+                                                       "max_episode_steps": 500})
+               ) -> JSONResponse:
 
     global envs
 
+    map_name = options.get("map_name", "4x4")
+    is_slippery = options.get("is_slippery", True)
+    max_episode_steps = options.get("max_episode_steps", 500)
+    env_tag = f"FrozenLake-{version}"
     if cidx in envs:
         env = envs[cidx]
 
         if env is not None:
             env.close()
 
         try:
-            env = gym.make(f"FrozenLake-{version}",
+            env = gym.make(id=env_tag,
                            map_name=map_name, is_slippery=is_slippery,
                            max_episode_steps=max_episode_steps)
             envs[cidx] = env
@@ -74,7 +80,7 @@ async def make(version: str = Body(default='v1'),
                                 detail=str(e))
     else:
         try:
-            env = gym.make(f"FrozenLake-{version}",
+            env = gym.make(id=env_tag,
                            map_name=map_name, is_slippery=is_slippery,
                            max_episode_steps=max_episode_steps)
             envs[cidx] = env

rest_api/gymnasium_envs/toy_text/taxi_env_api.py

@@ -53,10 +53,13 @@ async def close(cidx: int) -> JSONResponse:
 @taxi_router.post("/make")
 async def make(version: str = Body(default="v3"),
                cidx: int = Body(...),
-               max_episode_steps: int = Body(default=500)) -> JSONResponse:
+               options: dict[str, Any] = Body(default={"max_episode_steps": 500})
+               ) -> JSONResponse:
     global envs
     env_type = f"{ENV_NAME}-{version}"
 
+    max_episode_steps = options.get("max_episode_steps", 500)
+
     if cidx in envs:
         env = envs[cidx]