Fix decouple logging from console (#3765)

Author: izeigerman

sqlmesh/__init__.py

@@ -136,7 +136,6 @@ def format(self, record: logging.LogRecord) -> str:
 
 def configure_logging(
     force_debug: bool = False,
-    ignore_warnings: bool = False,
     write_to_stdout: bool = False,
     write_to_file: bool = True,
     log_limit: int = c.DEFAULT_LOG_LIMIT,
@@ -149,12 +148,11 @@ def configure_logging(
     level = logging.DEBUG if debug else logging.INFO
     logger.setLevel(level)
 
-    stdout_handler = logging.StreamHandler(sys.stdout)
-    stdout_handler.setFormatter(CustomFormatter())
-    stdout_handler.setLevel(
-        level if write_to_stdout else (logging.ERROR if ignore_warnings else logging.WARNING)
-    )
-    logger.addHandler(stdout_handler)
+    if write_to_stdout:
+        stdout_handler = logging.StreamHandler(sys.stdout)
+        stdout_handler.setFormatter(CustomFormatter())
+        stdout_handler.setLevel(level)
+        logger.addHandler(stdout_handler)
 
     log_file_dir = log_file_dir or c.DEFAULT_LOG_FILE_DIR
     log_path_prefix = Path(log_file_dir) / LOG_FILENAME_PREFIX

sqlmesh/cli/main.py

@@ -12,6 +12,7 @@
 from sqlmesh.cli import options as opt
 from sqlmesh.cli.example_project import ProjectTemplate, init_example_project
 from sqlmesh.core.analytics import cli_analytics
+from sqlmesh.core.console import configure_console, get_console
 from sqlmesh.core.config import load_configs
 from sqlmesh.core.context import Context
 from sqlmesh.utils.date import TimeLike
@@ -91,9 +92,8 @@ def cli(
 
     configs = load_configs(config, Context.CONFIG_TYPE, paths)
     log_limit = list(configs.values())[0].log_limit
-    configure_logging(
-        debug, ignore_warnings, log_to_stdout, log_limit=log_limit, log_file_dir=log_file_dir
-    )
+    configure_logging(debug, log_to_stdout, log_limit=log_limit, log_file_dir=log_file_dir)
+    configure_console(ignore_warnings=ignore_warnings)
 
     try:
         context = Context(
@@ -433,7 +433,7 @@ def plan(
     select_models = kwargs.pop("select_model") or None
     allow_destructive_models = kwargs.pop("allow_destructive_model") or None
     backfill_models = kwargs.pop("backfill_model") or None
-    context.console.verbose = verbose
+    setattr(get_console(), "verbose", verbose)
     context.plan(
         environment,
         restate_models=restate_models,

sqlmesh/core/config/connection.py

@@ -1776,7 +1776,7 @@ def _connection_config_validator(
     return parse_connection_config(v)
 
 
-connection_config_validator = field_validator(
+connection_config_validator: t.Callable = field_validator(
     "connection",
     "state_connection",
     "test_connection",

sqlmesh/core/config/root.py

@@ -4,7 +4,6 @@
 import re
 import typing as t
 import zlib
-import logging
 
 from pydantic import Field
 from sqlglot import exp
@@ -13,6 +12,7 @@
 
 from sqlmesh.cicd.config import CICDBotConfig
 from sqlmesh.core import constants as c
+from sqlmesh.core.console import get_console
 from sqlmesh.core.config import EnvironmentSuffixTarget
 from sqlmesh.core.config.base import BaseConfig, UpdateStrategy
 from sqlmesh.core.config.common import variables_validator, compile_regex_mapping
@@ -45,8 +45,6 @@
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import Self
 
-logger = logging.getLogger(__name__)
-
 
 class Config(BaseConfig):
     """An object used by a Context to configure your SQLMesh project.
@@ -175,13 +173,13 @@ def _normalize_and_validate_fields(cls, data: t.Any) -> t.Any:
                 )
 
         if "physical_schema_override" in data:
-            logger.warning(
-                "`physical_schema_override` is deprecated. Please use `physical_schema_mapping` instead"
+            get_console().log_warning(
+                "`physical_schema_override` is deprecated. Please use `physical_schema_mapping` instead."
             )
 
             if "physical_schema_mapping" in data:
                 raise ConfigError(
-                    "Only one of `physical_schema_override` and `physical_schema_mapping` can be specified"
+                    "Only one of `physical_schema_override` and `physical_schema_mapping` can be specified."
                 )
 
             physical_schema_override: t.Dict[str, str] = data.pop("physical_schema_override")

sqlmesh/core/config/scheduler.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import abc
-import logging
 import typing as t
 
 from pydantic import Field
@@ -10,7 +9,7 @@
 from sqlglot.helper import subclasses
 from sqlmesh.core.config.base import BaseConfig
 from sqlmesh.core.config.common import concurrent_tasks_validator
-from sqlmesh.core.console import Console
+from sqlmesh.core.console import Console, get_console
 from sqlmesh.core.plan import (
     AirflowPlanEvaluator,
     BuiltInPlanEvaluator,
@@ -32,8 +31,6 @@
 
 from sqlmesh.utils.config import sensitive_fields, excluded_fields
 
-logger = logging.getLogger(__name__)
-
 
 class SchedulerConfig(abc.ABC):
     """Abstract base class for Scheduler configurations."""
@@ -88,10 +85,10 @@ def create_state_sync(self, context: GenericContext) -> StateSync:
         ):
             # If we are using DuckDB, ensure that multithreaded mode gets enabled if necessary
             if warehouse_connection.concurrent_tasks > 1:
-                logger.warning(
+                get_console().log_warning(
                     "The duckdb state connection is configured for single threaded mode but the warehouse connection is configured for "
                     + f"multi threaded mode with {warehouse_connection.concurrent_tasks} concurrent tasks."
-                    + " This can cause SQLMesh to hang. Overriding the duckdb state connection config to use multi threaded mode"
+                    + " This can cause SQLMesh to hang. Overriding the duckdb state connection config to use multi threaded mode."
                 )
                 # this triggers multithreaded mode and has to happen before the engine adapter is created below
                 state_connection.concurrent_tasks = warehouse_connection.concurrent_tasks
@@ -109,7 +106,7 @@ def create_state_sync(self, context: GenericContext) -> StateSync:
             warehouse_connection, DuckDBConnectionConfig
         ):
             if not state_connection.is_recommended_for_state_sync:
-                logger.warning(
+                get_console().log_warning(
                     f"The {state_connection.type_} engine is not recommended for storing SQLMesh state in production deployments. Please see"
                     + " https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#state-connection for a list of recommended engines and more information."
                 )
@@ -205,7 +202,7 @@ def get_default_catalog(self, context: GenericContext) -> t.Optional[str]:
 
 
 def _max_snapshot_ids_per_request_validator(v: t.Any) -> t.Optional[int]:
-    logger.warning(
+    get_console().log_warning(
         "The `max_snapshot_ids_per_request` field is deprecated and will be removed in a future release."
     )
     return None