CM-27145 - Add setting of logger level for all loggers; fix stream of loggers; fix setting of logger level from env (#162)

Author: MarshalX

cycode/cli/code_scanner.py

@@ -33,13 +33,13 @@
     load_json,
 )
 from cycode.cli.utils.progress_bar import ProgressBarSection
-from cycode.cli.utils.progress_bar import logger as progress_bar_logger
 from cycode.cli.utils.scan_batch import run_parallel_batched_scan
 from cycode.cli.utils.scan_utils import set_issue_detected
 from cycode.cli.utils.string_utils import get_content_size, is_binary_content
 from cycode.cli.utils.task_timer import TimeoutAfter
 from cycode.cli.zip_file import InMemoryZip
 from cycode.cyclient import logger
+from cycode.cyclient.config import set_logging_level
 from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult
 
 if TYPE_CHECKING:
@@ -1399,9 +1399,7 @@ def perform_post_pre_receive_scan_actions(context: click.Context) -> None:
 
 def enable_verbose_mode(context: click.Context) -> None:
     context.obj['verbose'] = True
-    # TODO(MarshalX): rework setting the log level for loggers
-    logger.setLevel(logging.DEBUG)
-    progress_bar_logger.setLevel(logging.DEBUG)
+    set_logging_level(logging.DEBUG)
 
 
 def is_verbose_mode_requested_in_pre_receive_scan() -> bool:

cycode/cli/consts.py

@@ -112,8 +112,6 @@
 TIMEOUT_ENV_VAR_NAME = 'TIMEOUT'
 CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME = 'CYCODE_CLI_REQUEST_TIMEOUT'
 LOGGING_LEVEL_ENV_VAR_NAME = 'LOGGING_LEVEL'
-# use only for dev envs locally
-BATCH_SIZE_ENV_VAR_NAME = 'BATCH_SIZE'
 VERBOSE_ENV_VAR_NAME = 'CYCODE_CLI_VERBOSE'
 
 CYCODE_CONFIGURATION_DIRECTORY: str = '.cycode'

cycode/cli/main.py

@@ -22,8 +22,7 @@
 from cycode.cli.user_settings.user_settings_commands import add_exclusions, set_credentials
 from cycode.cli.utils import scan_utils
 from cycode.cli.utils.progress_bar import get_progress_bar
-from cycode.cli.utils.progress_bar import logger as progress_bar_logger
-from cycode.cyclient import logger
+from cycode.cyclient.config import set_logging_level
 from cycode.cyclient.cycode_client_base import CycodeClientBase
 from cycode.cyclient.models import UserAgentOptionScheme
 from cycode.cyclient.scan_config.scan_config_creator import create_scan_client
@@ -228,10 +227,8 @@ def main_cli(
 
     verbose = verbose or configuration_manager.get_verbose_flag()
     context.obj['verbose'] = verbose
-    # TODO(MarshalX): rework setting the log level for loggers
-    log_level = logging.DEBUG if verbose else logging.INFO
-    logger.setLevel(log_level)
-    progress_bar_logger.setLevel(log_level)
+    if verbose:
+        set_logging_level(logging.DEBUG)
 
     context.obj['output'] = output
     if output == 'json':

cycode/cyclient/config.py

@@ -6,17 +6,21 @@
 
 from cycode.cli import consts
 from cycode.cli.user_settings.configuration_manager import ConfigurationManager
+from cycode.cyclient.config_dev import DEV_MODE_ENV_VAR_NAME, DEV_TENANT_ID_ENV_VAR_NAME
 
-# set io encoding (for windows)
-from .config_dev import DEV_MODE_ENV_VAR_NAME, DEV_TENANT_ID_ENV_VAR_NAME
 
-sys.stdout.reconfigure(encoding='UTF-8')
-sys.stderr.reconfigure(encoding='UTF-8')
+def _set_io_encodings() -> None:
+    # set io encoding (for Windows)
+    sys.stdout.reconfigure(encoding='UTF-8')
+    sys.stderr.reconfigure(encoding='UTF-8')
+
+
+_set_io_encodings()
 
 # logs
 logging.basicConfig(
-    stream=sys.stdout,
-    level=logging.DEBUG,
+    stream=sys.stderr,
+    level=logging.INFO,
     format='%(asctime)s [%(name)s] %(levelname)s: %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S',
 )
@@ -34,19 +38,28 @@
     consts.TIMEOUT_ENV_VAR_NAME: 300,
     consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO,
     DEV_MODE_ENV_VAR_NAME: 'False',
-    consts.BATCH_SIZE_ENV_VAR_NAME: 20,
 }
 
 configuration = dict(DEFAULT_CONFIGURATION, **os.environ)
 
+_CREATED_LOGGERS = set()
+
 
 def get_logger(logger_name: Optional[str] = None) -> logging.Logger:
-    logger = logging.getLogger(logger_name)
-    level = _get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME)
-    level = level if level in logging._nameToLevel else int(level)
-    logger.setLevel(level)
+    config_level = _get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME)
+    level = logging.getLevelName(config_level)
+
+    new_logger = logging.getLogger(logger_name)
+    new_logger.setLevel(level)
+
+    _CREATED_LOGGERS.add(new_logger)
 
-    return logger
+    return new_logger
+
+
+def set_logging_level(level: int) -> None:
+    for created_logger in _CREATED_LOGGERS:
+        created_logger.setLevel(level)
 
 
 def _get_val_as_string(key: str) -> str:
@@ -66,15 +79,20 @@ def _get_val_as_int(key: str) -> Optional[int]:
     return None
 
 
-logger = get_logger('cycode cli')
+def _is_valid_url(url: str) -> bool:
+    try:
+        urlparse(url)
+        return True
+    except ValueError as e:
+        logger.warning(f'Invalid cycode api url: {url}, using default value', e)
+        return False
 
+
+logger = get_logger('cycode cli')
 configuration_manager = ConfigurationManager()
 
 cycode_api_url = configuration_manager.get_cycode_api_url()
-try:
-    urlparse(cycode_api_url)
-except ValueError as e:
-    logger.warning(f'Invalid cycode api url: {cycode_api_url}, using default value', e)
+if not _is_valid_url(cycode_api_url):
     cycode_api_url = consts.DEFAULT_CYCODE_API_URL
 
 timeout = _get_val_as_int(consts.CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME)
@@ -83,5 +101,3 @@ def _get_val_as_int(key: str) -> Optional[int]:
 
 dev_mode = _get_val_as_bool(DEV_MODE_ENV_VAR_NAME)
 dev_tenant_id = _get_val_as_string(DEV_TENANT_ID_ENV_VAR_NAME)
-batch_size = _get_val_as_int(consts.BATCH_SIZE_ENV_VAR_NAME)
-verbose = _get_val_as_bool(consts.VERBOSE_ENV_VAR_NAME)