[Trading] handle lazy markets loading exchanges trading

Author: GuillaumeDSM

packages/tentacles/Trading/Mode/simple_market_making_trading_mode/api/core.py

@@ -107,11 +107,25 @@ async def _fill_market_making_data_by_symbol(
                 elif dependency.symbol not in dependency_symbol_alias_by_symbol:
                     dependency_symbol_alias_by_symbol[dependency.symbol] = None
             available_symbols = set(exchange_manager.exchange.get_all_available_symbols(active_only=True))
-            symbols_to_skip_ticker_fetch = {
-                source.pair
-                for source in price_sources
-                if source.formula and source.pair not in available_symbols
-            }
+            lazy_load_markets = exchange_manager.exchange.get_option_value(
+                octobot_trading.enums.ExchangeClientOptions.LAZY_LOAD_MARKETS
+            )
+            if lazy_load_markets:
+                symbols_to_skip_ticker_fetch = {
+                    source.pair
+                    for source in price_sources
+                    if source.formula and source.pair not in available_symbols
+                }
+            else:
+                symbols_to_skip_ticker_fetch = {
+                    symbol for symbol in (set(symbols) | set(dependency_symbol_alias_by_symbol.keys()))
+                    if symbol not in available_symbols
+                }
+            if symbols_to_skip_ticker_fetch:
+                _get_logger().info(
+                    f"Ignored unavailable pairs on [{exchange_internal_name}]: "
+                    f"{sorted(symbols_to_skip_ticker_fetch)}"
+                )
             symbols_to_fetch = (set(symbols) | set(dependency_symbol_alias_by_symbol.keys())) - symbols_to_skip_ticker_fetch
             tickers = {
                 symbol: ticker
@@ -121,7 +135,11 @@ async def _fill_market_making_data_by_symbol(
             tickers, ticker_updater = await _fetch_tickers(
                 exchange_manager, tickers, list(symbols_to_fetch)
             )
-            if missing_tickers_to_fetch := [symbol for symbol in symbols_to_fetch if symbol not in tickers]:
+            if missing_tickers_to_fetch := [
+                symbol for symbol in symbols_to_fetch
+                if symbol not in tickers
+                and (lazy_load_markets or symbol in available_symbols)
+            ]:
                 try:
                     tickers.update(await ticker_updater.fetch_all_tickers(missing_tickers_to_fetch))
                 except octobot_trading.errors.NotSupported as err:
@@ -694,6 +712,7 @@ async def _get_price_and_predicted_order_book(
     books_by_symbol = {}
     for pair, reference_price in reference_price_by_pair.items():
         if not reference_price or reference_price.is_nan():
+            error_by_pair[pair] = _get_unsupported_pair_message(pair, mm_exchange)
             continue
         mm_data = mm_data_by_exchange[mm_exchange].get(pair)
         _adapt_volume_if_necessary(mm_data, reference_price)
@@ -742,6 +761,10 @@ def _get_missing_symbol_message(symbol: str, exchange: str) -> str:
     )
 
 
+def _get_unsupported_pair_message(pair: str, exchange: str) -> str:
+    return f"{pair} is not supported on {exchange}"
+
+
 def _format_format_market_making_volume(volume: typing.Union[dict, None], error: typing.Union[str, None]):
     return {
         constants.VOLUME_KEY: volume,

packages/tentacles/Trading/Mode/simple_market_making_trading_mode/tests/api/test_core.py

@@ -589,7 +589,7 @@ async def test_get_price_and_predicted_order_book_with_invalid_formula(profile_d
 
 
 async def test_get_price_and_predicted_order_book_with_error(profile_data_with_full_mm_config, mm_data_by_exchange):
-    """Test _get_price_and_predicted_order_book with error handling."""
+    """Empty reference price sources yield an unsupported-pair error."""
     profile_data_with_full_mm_config.tentacles[0].config[
         simple_market_making_trading.SimpleMarketMakingTradingMode.CONFIG_PAIR_SETTINGS
     ][0][simple_market_making_trading.SimpleMarketMakingTradingMode.REFERENCE_PRICE] = []
@@ -602,7 +602,9 @@ async def test_get_price_and_predicted_order_book_with_error(profile_data_with_f
 
     assert result == {
         "BTC/USDT": {
-            market_making_constants.ERROR_KEY: "BTC/USDT reference price on binance can't be computed from the following price sources: {}"
+            market_making_constants.ERROR_KEY: market_making_core._get_unsupported_pair_message(
+                "BTC/USDT", "binance"
+            ),
         }
     }
 
@@ -1059,21 +1061,42 @@ async def _mock_exchange_manager_context(exchange_manager):
     yield exchange_manager
 
 
+def _configure_lazy_load_markets(exchange_manager, lazy_load_markets: bool):
+    exchange_manager.exchange.get_option_value = mock.Mock(
+        side_effect=lambda option, **kwargs: (
+            lazy_load_markets
+            if option == trading_enums.ExchangeClientOptions.LAZY_LOAD_MARKETS
+            else False
+        )
+    )
+
+
+def _get_symbols_passed_to_fetch_tickers(fetch_tickers_mock) -> set:
+    return {
+        symbol
+        for call_args in fetch_tickers_mock.call_args_list
+        for symbol in call_args.args[2]
+    }
+
+
 async def _call_fill_market_making_data_by_symbol(
     price_sources,
     available_symbols,
     formula_init_patches=None,
+    lazy_load_markets=True,
 ):
     profile_data = _profile_data_for_market_making_fill()
     mm_data_by_symbol_by_exchange = {}
     exchange_manager = mock.Mock()
     exchange_manager.exchange.get_all_available_symbols = mock.Mock(
         return_value=available_symbols
     )
+    _configure_lazy_load_markets(exchange_manager, lazy_load_markets)
     ticker_updater = mock.Mock()
     ticker_updater.fetch_all_tickers = mock.AsyncMock(return_value={})
     ticker_cache = mock.Mock()
     ticker_cache.get_all_tickers = mock.Mock(return_value={})
+    fetch_tickers_mock = mock.AsyncMock(return_value=({}, ticker_updater))
 
     patches = [
         mock.patch.object(
@@ -1099,7 +1122,7 @@ async def _call_fill_market_making_data_by_symbol(
         mock.patch.object(
             market_making_core,
             "_fetch_tickers",
-            mock.AsyncMock(return_value=({}, ticker_updater)),
+            fetch_tickers_mock,
         ),
     ]
     if formula_init_patches:
@@ -1118,7 +1141,10 @@ async def _call_fill_market_making_data_by_symbol(
             auth=None,
         )
 
-    return mm_data_by_symbol_by_exchange, ticker_updater.fetch_all_tickers
+    return mm_data_by_symbol_by_exchange, {
+        "fetch_tickers": fetch_tickers_mock,
+        "fetch_all_tickers": ticker_updater.fetch_all_tickers,
+    }
 
 
 class TestFillMarketMakingDataBySymbol:
@@ -1141,12 +1167,13 @@ async def test_skips_ticker_fetch_for_unsupported_ref_price_symbol_with_formula(
             mock.patch.object(exchange_operators, "create_ohlcv_operators", return_value=[]),
             mock.patch.object(exchange_operators, "create_price_operators", return_value=[]),
         ]
-        mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol(
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
             price_sources,
             available_symbols={"BTC/ETH", "ETH/USDT"},
             formula_init_patches=formula_init_patches,
         )
 
+        fetch_all_tickers_mock = fetch_mocks["fetch_all_tickers"]
         assert fetch_all_tickers_mock.call_count == 0 or all(
             "BTC/USDT" not in call_args.args[0]
             for call_args in fetch_all_tickers_mock.call_args_list
@@ -1189,12 +1216,13 @@ async def test_still_fetches_formula_dependency_symbols(self):
                 return_value=exchange_operators.create_price_operators(exchange_manager, "BTC/USDT"),
             ),
         ]
-        mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol(
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
             price_sources,
             available_symbols={"BTC/ETH", "ETH/USDT"},
             formula_init_patches=formula_init_patches,
         )
 
+        fetch_all_tickers_mock = fetch_mocks["fetch_all_tickers"]
         assert fetch_all_tickers_mock.call_count == 1
         fetched_symbols = fetch_all_tickers_mock.call_args.args[0]
         assert "BTC/USDT" not in fetched_symbols
@@ -1203,6 +1231,7 @@ async def test_still_fetches_formula_dependency_symbols(self):
         assert set(mm_data_by_symbol_by_exchange["binance"]) == {"BTC/USDT", "BTC/ETH", "ETH/USDT"}
 
     async def test_fetches_ticker_for_unsupported_symbol_without_formula(self):
+        """Lazy-load exchanges still fetch direct pairs not listed in available_symbols."""
         price_sources = [
             advanced_reference_price_import.AdvancedPriceSource(
                 exchange="binance",
@@ -1212,13 +1241,55 @@ async def test_fetches_ticker_for_unsupported_symbol_without_formula(self):
                 formula="",
             )
         ]
-        mm_data_by_symbol_by_exchange, fetch_all_tickers_mock = await _call_fill_market_making_data_by_symbol(
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
             price_sources,
             available_symbols={"BTC/ETH", "ETH/USDT"},
+            lazy_load_markets=True,
         )
 
-        assert fetch_all_tickers_mock.call_count == 1
-        assert fetch_all_tickers_mock.call_args.args[0] == ["BTC/USDT"]
+        fetch_tickers_mock = fetch_mocks["fetch_tickers"]
+        assert fetch_tickers_mock.call_count == 1
+        assert fetch_tickers_mock.call_args.args[2] == ["BTC/USDT"]
+
+    async def test_skips_unavailable_direct_pair_on_cex(self):
+        price_sources = [
+            advanced_reference_price_import.AdvancedPriceSource(
+                exchange="binance",
+                pair="BTC/USDT",
+                time_frame=advanced_reference_price_import.DEFAULT_TIME_FRAME,
+                weight=decimal.Decimal("1.0"),
+                formula="",
+            )
+        ]
+        mm_data_by_symbol_by_exchange, fetch_mocks = await _call_fill_market_making_data_by_symbol(
+            price_sources,
+            available_symbols={"BTC/ETH", "ETH/USDT"},
+            lazy_load_markets=False,
+        )
+
+        assert "BTC/USDT" not in _get_symbols_passed_to_fetch_tickers(fetch_mocks["fetch_tickers"])
+        btc_usdt_data = mm_data_by_symbol_by_exchange["binance"]["BTC/USDT"]
+        assert btc_usdt_data.price.is_nan()
+
+    async def test_still_fetches_unavailable_direct_pair_on_lazy_load_exchange(self):
+        price_sources = [
+            advanced_reference_price_import.AdvancedPriceSource(
+                exchange="binance",
+                pair="BTC/USDT",
+                time_frame=advanced_reference_price_import.DEFAULT_TIME_FRAME,
+                weight=decimal.Decimal("1.0"),
+                formula="",
+            )
+        ]
+        _, fetch_mocks = await _call_fill_market_making_data_by_symbol(
+            price_sources,
+            available_symbols={"BTC/ETH", "ETH/USDT"},
+            lazy_load_markets=True,
+        )
+
+        fetch_tickers_mock = fetch_mocks["fetch_tickers"]
+        assert fetch_tickers_mock.call_count == 1
+        assert fetch_tickers_mock.call_args.args[2] == ["BTC/USDT"]
 
 
 def _mock_create_price_operators(prices_by_symbol: dict[str, float]):

packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py

@@ -651,12 +651,14 @@ async def get_price_ticker(self, symbol: str, **kwargs: dict) -> typing.Optional
                 return self.adapter.adapt_ticker(
                     await self.client.fetch_ticker(symbol, params=kwargs)
                 )
+        except ccxt.async_support.BadSymbol as err:
+            raise octobot_trading.errors.UnSupportedSymbolError(str(err)) from err
         except ccxt.async_support.NotSupported:
             raise octobot_trading.errors.NotSupported
         except ccxt.async_support.BaseError as e:
             raise octobot_trading.errors.FailedRequest(
                 f"Failed to get_price_ticker {html_util.get_html_summary_if_relevant(e)}"
-            )
+            ) from e
 
     @ccxt_client_util.converted_ccxt_common_errors
     async def get_all_currencies_price_ticker(

packages/trading/octobot_trading/exchanges/types/rest_exchange.py

@@ -582,6 +582,21 @@ async def load_markets_for_symbols(self, symbols: list[str]) -> list[dict]:
         )
         return loaded_markets
 
+    def _is_lazy_market_loaded(self, symbol: str) -> bool:
+        client_markets = self.connector.client.markets or {}
+        return symbol in client_markets
+
+    async def ensure_lazy_market_loaded(self, symbol: str) -> None:
+        if not self.lazy_load_markets():
+            return
+        if self._is_lazy_market_loaded(symbol):
+            return
+        await self.load_markets_for_symbols([symbol])
+
+    async def get_market_status_including_lazy_load(self, symbol, price_example=None, with_fixer=True):
+        await self.ensure_lazy_market_loaded(symbol)
+        return self.get_market_status(symbol, price_example=price_example, with_fixer=with_fixer)
+
     def get_market_status(self, symbol, price_example=None, with_fixer=True):
         """
         Override using get_fixed_market_status in exchange tentacle if the default market status is not as expected

packages/trading/octobot_trading/modes/modes_util.py

@@ -195,7 +195,7 @@ async def convert_with_market_or_limit_order(
 
     # get order quantity
     quantity = _get_available_or_target_quantity(exchange_mgr, symbol, order_type, price, asset_amount)
-    symbol_market = exchange_mgr.exchange.get_market_status(symbol, with_fixer=False)
+    symbol_market = await exchange_mgr.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False)
     created_orders = []
     for order_quantity, order_price in \
             trading_personal_data.decimal_check_and_adapt_order_details_if_necessary(

packages/trading/octobot_trading/modes/script_keywords/basic_keywords/price.py

@@ -63,5 +63,5 @@ async def get_price_with_offset(context, offset_input, use_delta_type_as_flat_va
             f"1.2, -0.222, -0.222d, @65100, 5%, e5%, e500"
         )
 
-    symbol_market = context.exchange_manager.exchange.get_market_status(context.symbol, with_fixer=False)
+    symbol_market = await context.exchange_manager.exchange.get_market_status_including_lazy_load(context.symbol, with_fixer=False)
     return personal_data.decimal_adapt_price(symbol_market, computed_price)

packages/trading/octobot_trading/personal_data/orders/order_factory.py

@@ -473,7 +473,7 @@ async def create_base_orders_and_associated_elements(
         current_price = await personal_data.get_up_to_date_price(
             self.exchange_manager, symbol=symbol, timeout=constants.ORDER_DATA_FETCHING_TIMEOUT
         )
-        symbol_market = self.exchange_manager.exchange.get_market_status(symbol, with_fixer=False)
+        symbol_market = await self.exchange_manager.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False)
         ctx = script_keywords.get_base_context_from_exchange_manager(self.exchange_manager, symbol)
             # market orders have no price
         computed_price = current_price if price is None else await self._get_computed_price(ctx, price)

packages/trading/octobot_trading/personal_data/orders/order_util.py

@@ -272,7 +272,7 @@ async def get_pre_order_data(exchange_manager, symbol: str, timeout: int = None,
                              portfolio_type=commons_constants.PORTFOLIO_AVAILABLE,
                              target_price=None):
     price = target_price or await get_up_to_date_price(exchange_manager, symbol, timeout=timeout)
-    symbol_market = exchange_manager.exchange.get_market_status(symbol, with_fixer=False)
+    symbol_market = await exchange_manager.exchange.get_market_status_including_lazy_load(symbol, with_fixer=False)
     currency_available, market_available, market_quantity = get_portfolio_amounts(
         exchange_manager, symbol, price, portfolio_type=portfolio_type
     )

packages/trading/octobot_trading/personal_data/orders/types/limit/limit_order.py

@@ -34,12 +34,12 @@ async def update_price_if_outdated(self):
         try:
             current_price = await self.exchange_manager.exchange_symbols_data.get_exchange_symbol_data(self.symbol) \
                 .prices_manager.get_mark_price(timeout=constants.CHAINED_ORDER_PRICE_FETCHING_TIMEOUT)
-            self._update_limit_price_if_necessary(current_price)
+            await self._update_limit_price_if_necessary(current_price)
         except asyncio.TimeoutError:
             # price can't be checked
             return
 
-    def _update_limit_price_if_necessary(self, current_price):
+    async def _update_limit_price_if_necessary(self, current_price):
         updated_price = self.origin_price
         if self.side is enums.TradeOrderSide.BUY:
             highest_accepted_buy_price = (
@@ -58,7 +58,7 @@ def _update_limit_price_if_necessary(self, current_price):
                 # => Increase it to the current price
                 updated_price = lowest_accepted_sell_price
         if self.origin_price != updated_price:
-            symbol_market = self.exchange_manager.exchange.get_market_status(self.symbol, with_fixer=False)
+            symbol_market = await self.exchange_manager.exchange.get_market_status_including_lazy_load(self.symbol, with_fixer=False)
             self.origin_price = decimal_order_adapter.decimal_adapt_price(symbol_market, updated_price)
 
     async def update_order_status(self, force_refresh=False):