Init gateways in background (#3762)

* Init gateways in background

* Revert log level changes

Author: r4victor

src/dstack/_internal/server/app.py

@@ -52,7 +52,7 @@
     volumes,
 )
 from dstack._internal.server.services.config import ServerConfigManager
-from dstack._internal.server.services.gateways import gateway_connections_pool, init_gateways
+from dstack._internal.server.services.gateways import gateway_connections_pool
 from dstack._internal.server.services.locking import advisory_lock_ctx
 from dstack._internal.server.services.projects import get_or_create_default_project
 from dstack._internal.server.services.proxy.deps import ServerProxyDependencyInjector
@@ -157,7 +157,7 @@ async def lifespan(app: FastAPI):
                         {"show_path": False},
                     )
                     await server_config_manager.apply_config(session=session, owner=admin)
-            await init_gateways(session=session)
+
     update_default_project(
         project_name=DEFAULT_PROJECT_NAME,
         url=SERVER_URL,

src/dstack/_internal/server/background/scheduled_tasks/__init__.py

@@ -1,9 +1,11 @@
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.date import DateTrigger
 from apscheduler.triggers.interval import IntervalTrigger
 
 from dstack._internal.server import settings
 from dstack._internal.server.background.scheduled_tasks.events import delete_events
 from dstack._internal.server.background.scheduled_tasks.gateways import (
+    init_gateways_in_background,
     process_gateways_connections,
 )
 from dstack._internal.server.background.scheduled_tasks.idle_volumes import (
@@ -34,6 +36,8 @@ def start_scheduled_tasks() -> AsyncIOScheduler:
     Start periodic tasks triggered by `apscheduler` at specific times/intervals.
     Suitable for tasks that run infrequently and don't need to lock rows for a long time.
     """
+    # DateTrigger() to init gateways immediately.
+    _scheduler.add_job(init_gateways_in_background, DateTrigger(), max_instances=1)
     _scheduler.add_job(process_probes, IntervalTrigger(seconds=3, jitter=1))
     _scheduler.add_job(collect_metrics, IntervalTrigger(seconds=10), max_instances=1)
     _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1)

src/dstack/_internal/server/background/scheduled_tasks/gateways.py

@@ -10,13 +10,19 @@
 from dstack._internal.server.services.gateways import (
     GatewayConnection,
     gateway_connections_pool,
+    init_gateways,
 )
 from dstack._internal.server.services.locking import advisory_lock_ctx
 from dstack._internal.utils.logging import get_logger
 
 logger = get_logger(__name__)
 
 
+async def init_gateways_in_background():
+    async with get_session_ctx() as session:
+        await init_gateways(session=session)
+
+
 async def process_gateways_connections():
     await _remove_inactive_connections()
     await _process_active_connections()