Merge pull request hummingbot#6808 from hummingbot/staging

sync / staging -> master for Hummingbot version 1.24.0

CONTRIBUTING.md

@@ -82,6 +82,8 @@ If the Foundation team requests changes, make more commits to your branch to add
 
 A minimum of 75% unit test coverage is required for all changes included in a pull request. However, some components, like UI components, are excluded from this validation.
 
+To run tests locally, run `make test` after activating the environment.
+
 To calculate the diff-coverage locally on your computer, run `make development-diff-cover` after running all tests.
 
 ## Checklist

Dockerfile

@@ -60,7 +60,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 # Create mount points
-RUN mkdir -p /home/hummingbot/conf /home/hummingbot/conf/connectors /home/hummingbot/conf/strategies /home/hummingbot/logs /home/hummingbot/data /home/hummingbot/certs /home/hummingbot/scripts
+RUN mkdir -p /home/hummingbot/conf /home/hummingbot/conf/connectors /home/hummingbot/conf/strategies /home/hummingbot/conf/scripts /home/hummingbot/logs /home/hummingbot/data /home/hummingbot/certs /home/hummingbot/scripts
 
 WORKDIR /home/hummingbot
 

bin/hummingbot.py

@@ -15,6 +15,7 @@
     load_client_config_map_from_file,
     write_config_to_yml,
 )
+from hummingbot.client.config.security import Security
 from hummingbot.client.hummingbot_application import HummingbotApplication
 from hummingbot.client.settings import AllConnectorSettings
 from hummingbot.client.ui import login_prompt
@@ -26,11 +27,13 @@
 
 
 class UIStartListener(EventListener):
-    def __init__(self, hummingbot_app: HummingbotApplication, is_script: Optional[bool] = False, is_quickstart: Optional[bool] = False):
+    def __init__(self, hummingbot_app: HummingbotApplication, is_script: Optional[bool] = False,
+                 script_config: Optional[dict] = None, is_quickstart: Optional[bool] = False):
         super().__init__()
         self._hb_ref: ReferenceType = ref(hummingbot_app)
         self._is_script = is_script
         self._is_quickstart = is_quickstart
+        self._script_config = script_config
 
     def __call__(self, _):
         asyncio.create_task(self.ui_start_handler())
@@ -46,10 +49,12 @@ async def ui_start_handler(self):
                 write_config_to_yml(hb.strategy_config_map, hb.strategy_file_name, hb.client_config_map)
             hb.start(log_level=hb.client_config_map.log_level,
                      script=hb.strategy_name if self._is_script else None,
+                     conf=self._script_config,
                      is_quickstart=self._is_quickstart)
 
 
 async def main_async(client_config_map: ClientConfigAdapter):
+    await Security.wait_til_decryption_done()
     await create_yml_files_legacy()
 
     # This init_logging() call is important, to skip over the missing config warnings.

bin/hummingbot_quickstart.py

@@ -40,6 +40,10 @@ def __init__(self):
                           type=str,
                           required=False,
                           help="Specify a file in `conf/` to load as the strategy config file.")
+        self.add_argument("--script-conf", "-c",
+                          type=str,
+                          required=False,
+                          help="Specify a file in `conf/scripts` to configure a script strategy.")
         self.add_argument("--config-password", "-p",
                           type=str,
                           required=False,
@@ -94,11 +98,13 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana
 
     strategy_config = None
     is_script = False
+    script_config = None
     if config_file_name is not None:
         hb.strategy_file_name = config_file_name
         if config_file_name.split(".")[-1] == "py":
             hb.strategy_name = hb.strategy_file_name
             is_script = True
+            script_config = args.script_conf if args.script_conf else None
         else:
             strategy_config = await load_strategy_config_map_from_file(
                 STRATEGIES_CONF_DIR_PATH / config_file_name
@@ -116,7 +122,8 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana
 
     # The listener needs to have a named variable for keeping reference, since the event listener system
     # uses weak references to remove unneeded listeners.
-    start_listener: UIStartListener = UIStartListener(hb, is_script=is_script, is_quickstart=True)
+    start_listener: UIStartListener = UIStartListener(hb, is_script=is_script, script_config=script_config,
+                                                      is_quickstart=True)
     hb.app.add_listener(HummingbotUIEvent.Start, start_listener)
 
     tasks: List[Coroutine] = [hb.run()]
@@ -135,6 +142,10 @@ def main():
     # variable.
     if args.config_file_name is None and len(os.environ.get("CONFIG_FILE_NAME", "")) > 0:
         args.config_file_name = os.environ["CONFIG_FILE_NAME"]
+
+    if args.script_conf is None and len(os.environ.get("SCRIPT_CONFIG", "")) > 0:
+        args.script_conf = os.environ["SCRIPT_CONFIG"]
+
     if args.config_password is None and len(os.environ.get("CONFIG_PASSWORD", "")) > 0:
         args.config_password = os.environ["CONFIG_PASSWORD"]
 

hummingbot/VERSION

@@ -1 +1 @@
-1.23.0
+1.24.0

hummingbot/client/command/create_command.py

@@ -1,10 +1,18 @@
 import asyncio
 import copy
+import importlib
+import inspect
 import os
 import shutil
+import sys
+from collections import OrderedDict
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Optional
 
+import yaml
+
+from hummingbot.client import settings
+from hummingbot.client.config.config_data_types import BaseClientModel
 from hummingbot.client.config.config_helpers import (
     ClientConfigAdapter,
     ConfigValidationError,
@@ -20,34 +28,81 @@
 )
 from hummingbot.client.config.config_var import ConfigVar
 from hummingbot.client.config.strategy_config_data_types import BaseStrategyConfigMap
-from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, required_exchanges
+from hummingbot.client.settings import SCRIPT_STRATEGY_CONFIG_PATH, STRATEGIES_CONF_DIR_PATH, required_exchanges
 from hummingbot.client.ui.completer import load_completer
 from hummingbot.core.utils.async_utils import safe_ensure_future
+from hummingbot.exceptions import InvalidScriptModule
 
 if TYPE_CHECKING:
     from hummingbot.client.hummingbot_application import HummingbotApplication  # noqa: F401
 
 
-class CreateCommand:
-    def create(self,  # type: HummingbotApplication
-               file_name):
-        if file_name is not None:
-            file_name = format_config_file_name(file_name)
-            if (STRATEGIES_CONF_DIR_PATH / file_name).exists():
-                self.notify(f"{file_name} already exists.")
-                return
+class OrderedDumper(yaml.SafeDumper):
+    pass
 
-        safe_ensure_future(self.prompt_for_configuration(file_name))
 
-    async def prompt_for_configuration(
-        self,  # type: HummingbotApplication
-        file_name,
-    ):
+class CreateCommand:
+    def create(self,  # type: HummingbotApplication
+               script_to_config: Optional[str] = None,):
         self.app.clear_input()
         self.placeholder_mode = True
         self.app.hide_input = True
         required_exchanges.clear()
+        if script_to_config is not None:
+            safe_ensure_future(self.prompt_for_configuration_v2(script_to_config))
+        else:
+            safe_ensure_future(self.prompt_for_configuration())
+
+    async def prompt_for_configuration_v2(self,  # type: HummingbotApplication
+                                          script_to_config: str):
+        try:
+            module = sys.modules.get(f"{settings.SCRIPT_STRATEGIES_MODULE}.{script_to_config}")
+            script_module = importlib.reload(module)
+            config_class = next((member for member_name, member in inspect.getmembers(script_module)
+                                 if inspect.isclass(member) and
+                                 issubclass(member, BaseClientModel) and member not in [BaseClientModel]))
+            config_map = ClientConfigAdapter(config_class.construct())
+
+            await self.prompt_for_model_config(config_map)
+            if not self.app.to_stop_config:
+                file_name = await self.save_config_strategy_v2(script_to_config, config_map)
+                self.notify(f"A new config file has been created: {file_name}")
+            self.app.change_prompt(prompt=">>> ")
+            self.app.input_field.completer = load_completer(self)
+            self.placeholder_mode = False
+            self.app.hide_input = False
+
+        except StopIteration:
+            raise InvalidScriptModule(f"The module {script_to_config} does not contain any subclass of BaseModel")
+
+    async def save_config_strategy_v2(self, strategy_name: str, config_instance: BaseClientModel):
+        file_name = await self.prompt_new_file_name(strategy_name, True)
+        if self.app.to_stop_config:
+            self.app.set_text("")
+            return
+
+        strategy_path = Path(SCRIPT_STRATEGY_CONFIG_PATH) / file_name
+        # Extract the ordered field names from the Pydantic model
+        field_order = list(config_instance.__fields__.keys())
+
+        # Use ordered field names to create an ordered dictionary
+        ordered_config_data = OrderedDict((field, getattr(config_instance, field)) for field in field_order)
+
+        # Add a representer to use the ordered dictionary and dump the YAML file
+        def _dict_representer(dumper, data):
+            return dumper.represent_dict(data.items())
+
+        OrderedDumper.add_representer(OrderedDict, _dict_representer)
 
+        # Write the configuration data to the YAML file
+        with open(strategy_path, 'w') as file:
+            yaml.dump(ordered_config_data, file, Dumper=OrderedDumper, default_flow_style=False)
+
+        return file_name
+
+    async def prompt_for_configuration(
+        self,  # type: HummingbotApplication
+    ):
         strategy = await self.get_strategy_name()
 
         if self.app.to_stop_config:
@@ -60,9 +115,9 @@ async def prompt_for_configuration(
         if isinstance(config_map, ClientConfigAdapter):
             await self.prompt_for_model_config(config_map)
             if not self.app.to_stop_config:
-                file_name = await self.save_config_to_file(file_name, config_map)
+                file_name = await self.save_config_to_file(config_map)
         elif config_map is not None:
-            file_name = await self.prompt_for_configuration_legacy(file_name, strategy, config_map)
+            file_name = await self.prompt_for_configuration_legacy(strategy, config_map)
         else:
             self.app.to_stop_config = True
 
@@ -107,7 +162,6 @@ async def prompt_for_model_config(
 
     async def prompt_for_configuration_legacy(
         self,  # type: HummingbotApplication
-        file_name,
         strategy: str,
         config_map: Dict,
     ):
@@ -132,12 +186,11 @@ async def prompt_for_configuration_legacy(
             self.app.set_text("")
             return
 
-        if file_name is None:
-            file_name = await self.prompt_new_file_name(strategy)
-            if self.app.to_stop_config:
-                self.restore_config_legacy(config_map, config_map_backup)
-                self.app.set_text("")
-                return
+        file_name = await self.prompt_new_file_name(strategy)
+        if self.app.to_stop_config:
+            self.restore_config_legacy(config_map, config_map_backup)
+            self.app.set_text("")
+            return
         self.app.change_prompt(prompt=">>> ")
         strategy_path = STRATEGIES_CONF_DIR_PATH / file_name
         template = get_strategy_template_path(strategy)
@@ -207,32 +260,32 @@ async def prompt_a_config_legacy(
 
     async def save_config_to_file(
         self,  # type: HummingbotApplication
-        file_name: Optional[str],
         config_map: ClientConfigAdapter,
     ) -> str:
-        if file_name is None:
-            file_name = await self.prompt_new_file_name(config_map.strategy)
-            if self.app.to_stop_config:
-                self.app.set_text("")
-                return
+        file_name = await self.prompt_new_file_name(config_map.strategy)
+        if self.app.to_stop_config:
+            self.app.set_text("")
+            return
         self.app.change_prompt(prompt=">>> ")
         strategy_path = Path(STRATEGIES_CONF_DIR_PATH) / file_name
         save_to_yml(strategy_path, config_map)
         return file_name
 
     async def prompt_new_file_name(self,  # type: HummingbotApplication
-                                   strategy):
+                                   strategy: str,
+                                   is_script: bool = False):
         file_name = default_strategy_file_path(strategy)
         self.app.set_text(file_name)
         input = await self.app.prompt(prompt="Enter a new file name for your configuration >>> ")
         input = format_config_file_name(input)
-        file_path = os.path.join(STRATEGIES_CONF_DIR_PATH, input)
+        conf_dir_path = STRATEGIES_CONF_DIR_PATH if not is_script else SCRIPT_STRATEGY_CONFIG_PATH
+        file_path = os.path.join(conf_dir_path, input)
         if input is None or input == "":
             self.notify("Value is required.")
-            return await self.prompt_new_file_name(strategy)
+            return await self.prompt_new_file_name(strategy, is_script)
         elif os.path.exists(file_path):
             self.notify(f"{input} file already exists, please enter a new name.")
-            return await self.prompt_new_file_name(strategy)
+            return await self.prompt_new_file_name(strategy, is_script)
         else:
             return input
 

hummingbot/client/command/start_command.py

@@ -8,11 +8,13 @@
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set
 
 import pandas as pd
+import yaml
 
 import hummingbot.client.settings as settings
 from hummingbot import init_logging
 from hummingbot.client.command.gateway_api_manager import GatewayChainApiManager
 from hummingbot.client.command.gateway_command import GatewayCommand
+from hummingbot.client.config.config_data_types import BaseClientModel
 from hummingbot.client.config.config_helpers import get_strategy_starter_file
 from hummingbot.client.config.config_validators import validate_bool
 from hummingbot.client.config.config_var import ConfigVar
@@ -59,15 +61,17 @@ def _strategy_uses_gateway_connector(self, required_exchanges: Set[str]) -> bool
     def start(self,  # type: HummingbotApplication
               log_level: Optional[str] = None,
               script: Optional[str] = None,
+              conf: Optional[str] = None,
               is_quickstart: Optional[bool] = False):
         if threading.current_thread() != threading.main_thread():
             self.ev_loop.call_soon_threadsafe(self.start, log_level, script)
             return
-        safe_ensure_future(self.start_check(log_level, script, is_quickstart), loop=self.ev_loop)
+        safe_ensure_future(self.start_check(log_level, script, conf, is_quickstart), loop=self.ev_loop)
 
     async def start_check(self,  # type: HummingbotApplication
                           log_level: Optional[str] = None,
                           script: Optional[str] = None,
+                          conf: Optional[str] = None,
                           is_quickstart: Optional[bool] = False):
 
         if self._in_start_check or (self.strategy_task is not None and not self.strategy_task.done()):
@@ -98,8 +102,8 @@ async def start_check(self,  # type: HummingbotApplication
 
         if script:
             file_name = script.split(".")[0]
-            self.strategy_file_name = file_name
             self.strategy_name = file_name
+            self.strategy_file_name = conf if conf else file_name
         elif not await self.status_check_all(notify_success=False):
             self.notify("Status checks failed. Start aborted.")
             self._in_start_check = False
@@ -196,20 +200,24 @@ async def start_check(self,  # type: HummingbotApplication
             self._mqtt.patch_loggers()
 
     def start_script_strategy(self):
-        script_strategy = self.load_script_class()
+        script_strategy, config = self.load_script_class()
         markets_list = []
         for conn, pairs in script_strategy.markets.items():
             markets_list.append((conn, list(pairs)))
         self._initialize_markets(markets_list)
-        self.strategy = script_strategy(self.markets)
+        if config:
+            self.strategy = script_strategy(self.markets, config)
+        else:
+            self.strategy = script_strategy(self.markets)
 
     def load_script_class(self):
         """
         Imports the script module based on its name (module file name) and returns the loaded script class
 
         :param script_name: name of the module where the script class is defined
         """
-        script_name = self.strategy_file_name
+        script_name = self.strategy_name
+        config = None
         module = sys.modules.get(f"{settings.SCRIPT_STRATEGIES_MODULE}.{script_name}")
         if module is not None:
             script_module = importlib.reload(module)
@@ -222,10 +230,25 @@ def load_script_class(self):
                                  member not in [ScriptStrategyBase, DirectionalStrategyBase]))
         except StopIteration:
             raise InvalidScriptModule(f"The module {script_name} does not contain any subclass of ScriptStrategyBase")
-        return script_class
+        if self.strategy_name != self.strategy_file_name:
+            try:
+                config_class = next((member for member_name, member in inspect.getmembers(script_module)
+                                    if inspect.isclass(member) and
+                                    issubclass(member, BaseClientModel) and member not in [BaseClientModel]))
+                config = config_class(**self.load_script_yaml_config(config_file_path=self.strategy_file_name))
+                script_class.init_markets(config)
+            except StopIteration:
+                raise InvalidScriptModule(f"The module {script_name} does not contain any subclass of BaseModel")
+
+        return script_class, config
+
+    @staticmethod
+    def load_script_yaml_config(config_file_path: str) -> dict:
+        with open(settings.SCRIPT_STRATEGY_CONFIG_PATH / config_file_path, 'r') as file:
+            return yaml.safe_load(file)
 
     def is_current_strategy_script_strategy(self) -> bool:
-        script_file_name = settings.SCRIPT_STRATEGIES_PATH / f"{self.strategy_file_name}.py"
+        script_file_name = settings.SCRIPT_STRATEGIES_PATH / f"{self.strategy_name}.py"
         return script_file_name.exists()
 
     async def start_market_making(self,  # type: HummingbotApplication