[Flow] add bot process keywords

Author: GuillaumeDSM

octobot/cli.py

@@ -15,6 +15,7 @@
 #  License along with OctoBot. If not, see <https://www.gnu.org/licenses/>.
 import argparse
 import os
+import pathlib
 import sys
 import multiprocessing
 import asyncio
@@ -30,6 +31,7 @@
     import octobot_commons.authentication as authentication
     import octobot_commons.constants as common_constants
     import octobot_commons.errors as errors
+    import octobot_commons.user_root_folder_provider as user_root_folder_provider
 
     import octobot_services.api as service_api
 
@@ -103,6 +105,14 @@ def _disable_interface_from_param(interface_identifier, param_value, logger):
             logger.info(interface_identifier.capitalize() + " interface disabled")
 
 
+def _set_user_root_from_cli(user_folder: str) -> None:
+    if not (user_folder and str(user_folder).strip()):
+        raise errors.ConfigError("User folder must be a non-empty path.")
+    if ".." in pathlib.PurePath(user_folder).parts:
+        raise errors.ConfigError("Invalid user folder: parent directory segments are not allowed.")
+    user_root_folder_provider.instance().set_root(os.path.normpath(user_folder))
+
+
 def _log_environment(logger):
     try:
         bot_type = "cloud" if constants.IS_CLOUD_ENV else "self-hosted"
@@ -115,10 +125,12 @@ def _log_environment(logger):
 
 def _create_configuration():
     config_path = configuration.get_user_config()
-    config = configuration.Configuration(config_path,
-                                         common_constants.USER_PROFILES_FOLDER,
-                                         constants.CONFIG_FILE_SCHEMA,
-                                         constants.PROFILE_FILE_SCHEMA)
+    config = configuration.Configuration(
+        config_path,
+        user_root_folder_provider.get_user_profiles_folder(),
+        constants.CONFIG_FILE_SCHEMA,
+        constants.PROFILE_FILE_SCHEMA,
+    )
     return config
 
 
@@ -297,7 +309,9 @@ def _load_or_create_tentacles(community_auth, config, logger):
     # add tentacles folder to Python path
     sys.path.append(os.path.realpath(os.getcwd()))
 
-    if os.path.isfile(tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_FILE_PATH):
+    if os.path.isfile(
+        user_root_folder_provider.get_user_reference_tentacle_config_file_path()
+    ):
         # when tentacles folder already exists
         config.load_profiles_if_possible_and_necessary()
         tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(
@@ -318,9 +332,13 @@ def start_octobot(args, default_config_file=None):
             print(constants.LONG_VERSION)
             return
 
-        # log folder can be overridden by the LOGS_FOLDER environment variable, 
-        # useful to run multiple bots from the same folder
-        logger = octobot_logger.init_logger(logs_folder=constants.LOGS_FOLDER)
+        user_folder = getattr(args, "user_folder", None)
+        if user_folder:
+            _set_user_root_from_cli(user_folder)
+
+        # log folder: --log-folder overrides default (from LOGS_FOLDER env at import + default "logs")
+        logs_folder = getattr(args, "log_folder", None) or constants.LOGS_FOLDER
+        logger = octobot_logger.init_logger(logs_folder=logs_folder)
         startup_messages = []
 
         # Version
@@ -384,6 +402,11 @@ def start_octobot(args, default_config_file=None):
                                         reset_trading_history=args.reset_trading_history,
                                         startup_messages=startup_messages)
 
+        if not args.backtesting:
+            path = getattr(args, "dump_state", None)
+            if path:
+                bot.dump_state_path = os.path.normpath(path)
+
         # set global bot instance
         commands.set_global_bot_instance(bot)
 
@@ -473,6 +496,18 @@ def octobot_parser(parser, default_config_file=None):
                              'When disabled, the backtesting run will not be interrupted during execution',
                         action='store_true')
     parser.add_argument('-r', '--risk', type=float, help='Force a specific risk configuration (between 0 and 1).')
+    parser.add_argument(
+        '--user-folder',
+        type=str,
+        default=None,
+        help='User data root (config, profiles, reference tentacles). Relative to the current working directory.',
+    )
+    parser.add_argument(
+        '--log-folder',
+        type=str,
+        default=None,
+        help='Log files directory. When set, overrides the LOGS_FOLDER environment variable and default "logs".',
+    )
     parser.add_argument('-nw', '--no_web', help="Don't start OctoBot web interface.",
                         action='store_true')
     parser.add_argument('-nl', '--no_logs', help="Disable OctoBot logs in backtesting.",
@@ -486,6 +521,13 @@ def octobot_parser(parser, default_config_file=None):
                                             " exchanges configuration in your config.json without using any interface "
                                             "(ie the web interface that handle encryption automatically).",
                         action='store_true')
+    parser.add_argument(
+        "--dump-state",
+        type=str,
+        default=None,
+        help="Absolute path of the JSON file where OctoBot periodically writes ProcessBotState (liveness, "
+        "next to the user config directory). Omitted in normal use; spawned DSL children pass this explicitly.",
+    )
     parser.add_argument('--identifier', help="OctoBot community identifier.", type=str, nargs=1)
     parser.add_argument('-o', '--strategy_optimizer', help='Start Octobot strategy optimizer. This mode will make '
                                                            'octobot play backtesting scenarii located in '
@@ -603,6 +645,8 @@ def start_background_octobot_with_args(
     in_subprocess=False,
     reset_trading_history=False,
     default_config_file=None,
+    user_folder=None,
+    log_folder=None,
 ):
     if backtesting_files is None:
         backtesting_files = []
@@ -621,7 +665,9 @@ def start_background_octobot_with_args(
                               enable_backtesting_timeout=enable_backtesting_timeout,
                               simulate=simulate,
                               risk=risk,
-                              reset_trading_history=reset_trading_history)
+                              reset_trading_history=reset_trading_history,
+                              user_folder=user_folder,
+                              log_folder=log_folder)
     if in_subprocess:
         bot_process = multiprocessing.Process(target=start_octobot, args=(args, default_config_file))
         bot_process.start()

octobot/commands.py

@@ -39,6 +39,8 @@
 import octobot.community.tentacles_packages as community_tentacles_packages
 import octobot.configuration_manager as configuration_manager
 
+import octobot.storage.process_bot_state_dumper as process_bot_state_dumper
+
 COMMANDS_LOGGER_NAME = "Commands"
 IGNORED_COMMAND_WHEN_RESTART = ["-u", "--update"]
 
@@ -324,6 +326,12 @@ async def start_bot(bot, logger, catch=False):
             await bot.initialize()
         except asyncio.CancelledError:
             logger.info("Core engine tasks cancelled.")
+        else:
+            if bot.dump_state_path:
+
+                bot._process_bot_state_dump_task = asyncio.create_task(
+                    process_bot_state_dumper.run_periodic_dump_loop(bot.dump_state_path, logger, bot)
+                )
 
     except Exception as e:
         logger.exception(e)

octobot/configuration_manager.py

@@ -20,6 +20,7 @@
 import octobot.constants as constants
 import octobot_commons.configuration as configuration
 import octobot_commons.constants as common_constants
+import octobot_commons.user_root_folder_provider as user_root_folder_provider
 import octobot_commons.logging as logging
 import octobot_commons.json_util as json_util
 import octobot_tentacles_manager.constants as tentacles_manager_constants
@@ -123,8 +124,9 @@ def init_config(
     :param from_config_file: the default config file path
     """
     try:
-        if not os.path.exists(common_constants.USER_FOLDER):
-            os.makedirs(common_constants.USER_FOLDER)
+        user_root = user_root_folder_provider.get_user_root_folder()
+        if not os.path.exists(user_root):
+            os.makedirs(user_root)
 
         shutil.copyfile(from_config_file, config_file)
     except Exception as global_exception:
@@ -169,35 +171,48 @@ def get_default_tentacles_url(version=None):
 
 def get_user_local_config_file():
     try:
-        import octobot_commons.constants as commons_constants
-        return f"{commons_constants.USER_FOLDER}/logging_config.ini"
+        import octobot_commons.user_root_folder_provider as user_root_folder_provider
+
+        return os.path.join(
+            user_root_folder_provider.get_user_root_folder(), "logging_config.ini"
+        )
     except ImportError:
         return None
 
 
 def load_default_tentacles_config(profile_folder):
-    if os.path.isdir(tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_PATH):
-        shutil.copyfile(tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_FILE_PATH,
-                        os.path.join(profile_folder, tentacles_manager_constants.constants.CONFIG_TENTACLES_FILE))
-        shutil.copytree(tentacles_manager_constants.USER_REFERENCE_TENTACLE_SPECIFIC_CONFIG_PATH,
-                        os.path.join(profile_folder, tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER))
+    ref_path = user_root_folder_provider.get_user_reference_tentacle_config_path()
+    ref_file = user_root_folder_provider.get_user_reference_tentacle_config_file_path()
+    ref_spec = user_root_folder_provider.get_user_reference_tentacle_specific_config_path()
+    if os.path.isdir(ref_path):
+        shutil.copyfile(
+            ref_file,
+            os.path.join(profile_folder, tentacles_manager_constants.constants.CONFIG_TENTACLES_FILE),
+        )
+        shutil.copytree(
+            ref_spec,
+            os.path.join(profile_folder, tentacles_manager_constants.TENTACLES_SPECIFIC_CONFIG_FOLDER),
+        )
 
 
 def migrate_from_previous_config(config):
     logger = logging.get_logger(LOGGER_NAME)
     # migrate tentacles configuration if necessary
-    previous_tentacles_config = os.path.join(common_constants.USER_FOLDER, "tentacles_config")
-    previous_tentacles_config_save = os.path.join(common_constants.USER_FOLDER, "tentacles_config.back")
-    if os.path.isdir(previous_tentacles_config) and \
-            not os.path.isdir(tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_PATH):
+    user_root = user_root_folder_provider.get_user_root_folder()
+    ref_tent_path = user_root_folder_provider.get_user_reference_tentacle_config_path()
+    previous_tentacles_config = os.path.join(user_root, "tentacles_config")
+    previous_tentacles_config_save = os.path.join(user_root, "tentacles_config.back")
+    if os.path.isdir(previous_tentacles_config) and not os.path.isdir(ref_tent_path):
         logger.info(
             f"Updating your tentacles configuration located in {previous_tentacles_config} into the new format. "
             f"A save of your previous tentacles config is available in {previous_tentacles_config_save}")
-        shutil.copytree(previous_tentacles_config,
-                        tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_PATH)
+        shutil.copytree(previous_tentacles_config, ref_tent_path)
         shutil.move(previous_tentacles_config, previous_tentacles_config_save)
         load_default_tentacles_config(
-            os.path.join(common_constants.USER_PROFILES_FOLDER, common_constants.DEFAULT_PROFILE)
+            os.path.join(
+                user_root_folder_provider.get_user_profiles_folder(),
+                common_constants.DEFAULT_PROFILE,
+            )
         )
     # migrate global configuration if necessary
     config_path = configuration.get_user_config()

octobot/constants.py

@@ -202,6 +202,16 @@
 # logs
 DEFAULT_LOGS_FOLDER = "logs"
 LOGS_FOLDER = os.getenv("LOGS_FOLDER", DEFAULT_LOGS_FOLDER)
+
+# Web automation: child process sets OCTOBOT_WEB_API_KEY
+ENV_WEB_API_KEY = "OCTOBOT_WEB_API_KEY"
+WEB_API_KEY_HEADER = "X-Octobot-Api-Key"
+# Process bot state JSON next to user config (--dump-state); liveness for run_octobot_process
+PROCESS_BOT_STATE_FILE_NAME = "process_bot_state.json"
+ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = "OCTOBOT_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS"
+PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = float(
+    os.getenv(ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "30")
+)
 FORCED_LOG_LEVEL = os.getenv("FORCED_LOG_LEVEL", "")
 ENV_TRADING_ENABLE_DEBUG_LOGS = os_util.parse_boolean_environment_var("ENV_TRADING_ENABLE_DEBUG_LOGS", "False")
 

octobot/logger.py

@@ -38,6 +38,7 @@
 
 import octobot.constants as constants
 import octobot.configuration_manager as configuration_manager
+import octobot_commons.user_root_folder_provider as user_root_folder_provider
 
 BOT_CHANNEL_LOGGER = None
 LOGGER_PRIORITY_LEVEL = channel_enums.ChannelConsumerPriorityLevels.OPTIONAL.value
@@ -51,7 +52,7 @@ def _log_uncaught_exceptions(ex_cls, ex, tb):
 def init_logger(logs_folder: str = constants.DEFAULT_LOGS_FOLDER):
     try:
         if not os.path.exists(logs_folder):
-            os.mkdir(logs_folder)
+            os.makedirs(logs_folder)
         _load_logger_config(logs_folder)
         init_bot_channel_logger()
     except KeyError:
@@ -88,8 +89,9 @@ def _load_logger_config(logs_folder: str):
     try:
         # use local logging file to allow users to customize the log level
         if not os.path.isfile(configuration_manager.get_user_local_config_file()):
-            if not os.path.exists(commons_constants.USER_FOLDER):
-                os.mkdir(commons_constants.USER_FOLDER)
+            user_root = user_root_folder_provider.get_user_root_folder()
+            if not os.path.exists(user_root):
+                os.mkdir(user_root)
             shutil.copyfile(constants.LOGGING_CONFIG_FILE, configuration_manager.get_user_local_config_file())
         logging.config.fileConfig(configuration_manager.get_user_local_config_file())
         logger = logging.getLogger("Logging Configuration")

octobot/octobot.py

@@ -105,6 +105,9 @@ def __init__(self, config: configuration.Configuration, community_authenticator=
         self.initializer = initializer.Initializer(self)
         self.task_manager = task_manager.TaskManager(self)
         self._init_metadata_run_task = None
+        # optional path for periodic ProcessBotState JSON (see cli --dump-state)
+        self.dump_state_path = None
+        self._process_bot_state_dump_task = None
 
         # Producers
         self.exchange_producer = None
@@ -211,6 +214,9 @@ async def stop(self):
             self.logger.debug("Stopping ...")
             if self._init_metadata_run_task is not None and not self._init_metadata_run_task.done():
                 self._init_metadata_run_task.cancel()
+            if self._process_bot_state_dump_task is not None and not self._process_bot_state_dump_task.done():
+                self._process_bot_state_dump_task.cancel()
+            self._process_bot_state_dump_task = None
             signals.SignalPublisher.instance().stop()
             if self.evaluator_producer is not None:
                 await self.evaluator_producer.stop()