feat: quiet-logs flag (#2072)

* feat: quiet-logs flag

* fix: include quiet_logs in config. make it a flag

* fix: initiate self.quiet_logs in the Config constructor

---------

Co-authored-by: Yosef Arbiv <yosef@elementary-data.com>
Co-authored-by: Itamar Hartstein <haritamar@gmail.com>

Author: 3 people

elementary/cli/cli.py

@@ -29,6 +29,13 @@ def get_log_path(ctx):
         return os.path.join(target_path, "edr.log")
 
 
+def get_quiet_logs(ctx):
+    try:
+        return "--quiet-logs" in ctx.args
+    except (ValueError, AttributeError):
+        return False
+
+
 class ElementaryCLI(click.MultiCommand):
     _CMD_MAP = {
         "monitor": monitor,
@@ -57,12 +64,14 @@ def format_help(self, ctx, formatter):
 
     def invoke(self, ctx: click.Context) -> Any:
         files_target_path = get_log_path(ctx)
-        set_root_logger_handlers("elementary", files_target_path)
-        click.echo(
-            "Any feedback and suggestions are welcomed! join our community here - "
-            "https://bit.ly/slack-elementary\n"
-        )
-        logger.info(f"Running with edr={package.get_package_version()}")
+        quiet_logs = get_quiet_logs(ctx)
+        set_root_logger_handlers("elementary", files_target_path, quiet_logs=quiet_logs)
+        if not quiet_logs:
+            click.echo(
+                "Any feedback and suggestions are welcomed! join our community here - "
+                "https://bit.ly/slack-elementary\n"
+            )
+            logger.info(f"Running with edr={package.get_package_version()}")
         return super().invoke(ctx)
 
 

elementary/config/config.py

@@ -75,6 +75,7 @@ def __init__(
         env: str = DEFAULT_ENV,
         run_dbt_deps_if_needed: Optional[bool] = None,
         project_name: Optional[str] = None,
+        quiet_logs: Optional[bool] = None,
     ):
         self.config_dir = config_dir
         self.profiles_dir = profiles_dir
@@ -217,6 +218,10 @@ def __init__(
             "disable_elementary_logo_print", False
         )
 
+        self.quiet_logs = self._first_not_none(
+            quiet_logs, config.get("quiet_logs"), False
+        )
+
     def _load_configuration(self) -> dict:
         if not os.path.exists(self.config_dir):
             os.makedirs(self.config_dir)

elementary/monitor/cli.py

@@ -149,6 +149,12 @@ def decorator(func):
             else "DEPRECATED! Please use --filters instead! - Filter the alerts by tags:<TAGS> / owners:<OWNERS> / models:<MODELS> / "
             "statuses:<warn/fail/error/skipped> / resource_types:<model/test>.",
         )(func)
+        func = click.option(
+            "--quiet-logs",
+            is_flag=True,
+            default=False,
+            help="Minimize INFO level logs. Only WARNING and above will be shown.",
+        )(func)
         return func
 
     return decorator
@@ -324,12 +330,14 @@ def monitor(
     excludes,
     teams_webhook,
     maximum_columns_in_alert_samples,
+    quiet_logs,
 ):
     """
     Get alerts on failures in dbt jobs.
     """
     if ctx.invoked_subcommand is not None:
         return
+
     if deprecated_slack_webhook is not None:
         click.secho(
             '\n"-s" is deprecated and won\'t be supported in the near future.\n'
@@ -356,6 +364,7 @@ def monitor(
         report_url=report_url,
         teams_webhook=teams_webhook,
         maximum_columns_in_alert_samples=maximum_columns_in_alert_samples,
+        quiet_logs=quiet_logs,
     )
     anonymous_tracking = AnonymousCommandLineTracking(config)
     anonymous_tracking.set_env("use_select", bool(select))
@@ -450,6 +459,7 @@ def report(
     env,
     select,
     target_path,
+    quiet_logs,
 ):
     """
     Generate a local observability report of your warehouse.
@@ -463,6 +473,7 @@ def report(
         target_path,
         dbt_quoting=dbt_quoting,
         env=env,
+        quiet_logs=quiet_logs,
     )
     anonymous_tracking = AnonymousCommandLineTracking(config)
     anonymous_tracking.set_env("use_select", bool(select))
@@ -680,6 +691,7 @@ def send_report(
     disable,
     include,
     target_path,
+    quiet_logs,
 ):
     """
     Generate and send the report to an external platform.
@@ -722,6 +734,7 @@ def send_report(
         report_url=report_url,
         env=env,
         project_name=project_name,
+        quiet_logs=quiet_logs,
     )
     anonymous_tracking = AnonymousCommandLineTracking(config)
     anonymous_tracking.set_env("use_select", bool(select))

elementary/monitor/fetchers/alerts/alerts.py

@@ -1,6 +1,8 @@
 import json
 from typing import Dict, List, Optional
 
+import click
+
 from elementary.clients.dbt.base_dbt_runner import BaseDbtRunner
 from elementary.clients.fetcher.fetcher import FetcherClient
 from elementary.config.config import Config
@@ -29,7 +31,7 @@ def skip_alerts(
     ):
         alert_ids = [alert.id for alert in alerts_to_skip]
         alert_ids_chunks = self._split_list_to_chunks(alert_ids)
-        logger.info("Update skipped alerts")
+        click.echo(f"Update skipped alerts ({len(alerts_to_skip)})")
         for alert_ids_chunk in alert_ids_chunks:
             self.dbt_runner.run(
                 select="elementary_cli.update_alerts.update_skipped_alerts",
@@ -60,7 +62,7 @@ def query_last_alert_times(
 
     def update_sent_alerts(self, alert_ids: List[str]) -> None:
         alert_ids_chunks = self._split_list_to_chunks(alert_ids)
-        logger.info("Update sent alerts")
+        click.echo(f"Update sent alerts ({len(alert_ids)})")
         for alert_ids_chunk in alert_ids_chunks:
             self.dbt_runner.run(
                 select="elementary_cli.update_alerts.update_sent_alerts",

elementary/utils/log.py

@@ -30,10 +30,15 @@ def format(self, record):
 ROTATION_BACKUP_COUNT = 4
 
 
-def get_console_handler():
+def get_console_handler(quiet_logs: bool = False):
     console_handler = logging.StreamHandler(sys.stdout)
     console_handler.setFormatter(FORMATTER)
-    console_handler.setLevel(logging.DEBUG if is_debug() else logging.INFO)
+    if is_debug():
+        console_handler.setLevel(logging.DEBUG)
+    elif quiet_logs:
+        console_handler.setLevel(logging.WARNING)
+    else:
+        console_handler.setLevel(logging.INFO)
     return console_handler
 
 
@@ -55,7 +60,14 @@ def get_logger(logger_name):
     return logger
 
 
-def set_root_logger_handlers(logger_name, files_target_path):
+def set_root_logger_handlers(logger_name, files_target_path, quiet_logs: bool = False):
     logger = logging.getLogger(logger_name)
-    logger.addHandler(get_console_handler())
+
+    # Disable propagation to root logger to avoid duplicate logs
+    logger.propagate = False
+
+    logger.addHandler(get_console_handler(quiet_logs=quiet_logs))
     logger.addHandler(get_file_handler(files_target_path))
+
+    # Set logger level to DEBUG so it doesn't filter messages (handler will filter)
+    logger.setLevel(logging.DEBUG)