fix: initialize separate logger for each module

Author: sgoral-splunk

solnlib/conf_manager.py

@@ -36,6 +36,9 @@
     InvalidHostnameError,
 )
 
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
 __all__ = [
     "ConfFile",
     "ConfManager",
@@ -338,7 +341,7 @@ def delete(self, stanza_name: str):
         try:
             self._conf.delete(stanza_name)
         except KeyError:
-            logging.error(
+            logger.error(
                 "Delete stanza: %s error: %s.", stanza_name, traceback.format_exc()
             )
             raise ConfStanzaNotExistException(

solnlib/file_monitor.py

@@ -24,6 +24,10 @@
 import traceback
 from typing import Any, Callable, List
 
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
+
 __all__ = ["FileChangesChecker", "FileMonitor"]
 
 
@@ -45,7 +49,8 @@ def __init__(self, callback: Callable[[List[str]], Any], files: List):
             try:
                 self.file_mtimes[k] = op.getmtime(k)
             except OSError:
-                logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}")
+                logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}")  # deprecated
+                logger.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}")
 
     def check_changes(self) -> bool:
         """Check files change.
@@ -56,7 +61,7 @@ def check_changes(self) -> bool:
         Returns:
             True if files changed else False
         """
-        logging.debug(f"Checking files={self._files}")
+        logger.debug(f"Checking files={self._files}")
         file_mtimes = self.file_mtimes
         changed_files = []
         for f, last_mtime in list(file_mtimes.items()):
@@ -65,7 +70,7 @@ def check_changes(self) -> bool:
                 if current_mtime != last_mtime:
                     file_mtimes[f] = current_mtime
                     changed_files.append(f)
-                    logging.info(f"Detect {f} has changed", f)
+                    logger.info(f"Detect {f} has changed", f)
             except OSError:
                 pass
         if changed_files:

solnlib/modular_input/checkpointer.py

@@ -19,7 +19,6 @@
 
 import base64
 import json
-import logging
 import os
 import os.path as op
 import traceback
@@ -31,6 +30,9 @@
 
 from solnlib import _utils, utils
 
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
 __all__ = ["CheckpointerException", "KVStoreCheckpointer", "FileCheckpointer"]
 
 
@@ -178,7 +180,7 @@ def get(self, key: str) -> Optional[Any]:
             record = self._collection_data.query_by_id(key)
         except binding.HTTPError as e:
             if e.status != 404:
-                logging.error(f"Get checkpoint failed: {traceback.format_exc()}.")
+                logger.error(f"Get checkpoint failed: {traceback.format_exc()}.")
                 raise
             return None
         return json.loads(record["state"])
@@ -199,7 +201,7 @@ def delete(self, key: str) -> None:
             self._collection_data.delete_by_id(key)
         except binding.HTTPError as e:
             if e.status != 404:
-                logging.error(f"Delete checkpoint failed: {traceback.format_exc()}.")
+                logger.error(f"Delete checkpoint failed: {traceback.format_exc()}.")
                 raise
 
 

solnlib/modular_input/modinput.py

@@ -19,7 +19,9 @@
 import traceback
 
 import solnlib.splunkenv as sp
-import logging
+
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
 
 
 def _parse_modinput_configs(root, outer_block, inner_block):
@@ -62,7 +64,7 @@ def _parse_modinput_configs(root, outer_block, inner_block):
 
     confs = root.getElementsByTagName(outer_block)
     if not confs:
-        logging.error("Invalid config, missing %s section", outer_block)
+        logger.error("Invalid config, missing %s section", outer_block)
         raise Exception(f"Invalid config, missing {outer_block} section")
 
     configs = []
@@ -71,7 +73,7 @@ def _parse_modinput_configs(root, outer_block, inner_block):
         config = {}
         stanza_name = stanza.getAttribute("name")
         if not stanza_name:
-            logging.error("Invalid config, missing name")
+            logger.error("Invalid config, missing name")
             raise Exception("Invalid config, missing name")
 
         config["name"] = stanza_name
@@ -107,13 +109,13 @@ def parse_modinput_configs(config_str):
     for tag in meta_configs.keys():
         nodes = doc.getElementsByTagName(tag)
         if not nodes:
-            logging.error("Invalid config, missing %s section", tag)
+            logger.error("Invalid config, missing %s section", tag)
             raise Exception("Invalid config, missing %s section", tag)
 
         if nodes[0].firstChild and nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE:
             meta_configs[tag] = nodes[0].firstChild.data
         else:
-            logging.error("Invalid config, expect text ndoe")
+            logger.error("Invalid config, expect text ndoe")
             raise Exception("Invalid config, expect text ndoe")
 
     if doc.nodeName == "input":
@@ -140,7 +142,7 @@ def get_modinput_configs_from_cli(modinput, modinput_stanza=None):
         cli, stdout=subprocess.PIPE, stderr=subprocess.PIPE
     ).communicate()
     if err:
-        logging.error("Failed to get modinput configs with error: %s", err)
+        logger.error("Failed to get modinput configs with error: %s", err)
         return None, None
     else:
         return parse_modinput_configs(out)
@@ -152,7 +154,7 @@ def get_modinput_config_str_from_stdin():
     try:
         return sys.stdin.read(5000)
     except Exception:
-        logging.error(traceback.format_exc())
+        logger.error(traceback.format_exc())
         raise
 
 

solnlib/modular_input/modular_input.py

@@ -16,7 +16,6 @@
 
 """This module provides a base class of Splunk modular input."""
 
-import logging
 import sys
 import traceback
 from abc import ABCMeta, abstractmethod
@@ -35,6 +34,9 @@
 from ..orphan_process_monitor import OrphanProcessMonitor
 from . import checkpointer, event_writer
 
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
 __all__ = ["ModularInputException", "ModularInput"]
 
 
@@ -196,7 +198,7 @@ def _create_checkpointer(self):
                     port=self.server_port,
                 )
             except binding.HTTPError:
-                logging.error(
+                logger.error(
                     "Failed to init kvstore checkpointer: %s.", traceback.format_exc()
                 )
                 raise
@@ -234,7 +236,7 @@ def _create_event_writer(self):
                     global_settings_schema=self.hec_global_settings_schema,
                 )
             except binding.HTTPError:
-                logging.error(
+                logger.error(
                     "Failed to init HECEventWriter: %s.", traceback.format_exc()
                 )
                 raise
@@ -464,10 +466,10 @@ def execute(self):
                 else:
                     self.config_name = list(input_definition["inputs"].keys())[0]
                 self.do_run(input_definition["inputs"])
-                logging.info("Modular input: %s exit normally.", self.name)
+                logger.info("Modular input: %s exit normally.", self.name)
                 return 0
             except Exception:
-                logging.error(
+                logger.error(
                     "Modular input: %s exit with exception: %s.",
                     self.name,
                     traceback.format_exc(),
@@ -490,7 +492,7 @@ def execute(self):
                 self.do_validation(validation_definition["parameters"])
                 return 0
             except Exception as e:
-                logging.error(
+                logger.error(
                     "Modular input: %s validate arguments with exception: %s.",
                     self.name,
                     traceback.format_exc(),
@@ -501,7 +503,7 @@ def execute(self):
                 sys.stderr.flush()
                 return 1
         else:
-            logging.error(
+            logger.error(
                 'Modular input: %s run with invalid arguments: "%s".',
                 self.name,
                 " ".join(sys.argv[1:]),

solnlib/rest.py

@@ -23,18 +23,21 @@
 
 import logging
 
+from solnlib.log import Logs, log_exception
+
+logger = Logs().get_logger(__name__)
+
 
 def splunkd_request(
-    splunkd_uri,
-    session_key,
-    method="GET",
-    headers=None,
-    data=None,
-    timeout=300,
-    retry=1,
-    verify=False,
+        splunkd_uri,
+        session_key,
+        method="GET",
+        headers=None,
+        data=None,
+        timeout=300,
+        retry=1,
+        verify=False,
 ) -> Optional[requests.Response]:
-
     headers = headers if headers is not None else {}
     headers["Authorization"] = f"Splunk {session_key}"
     content_type = headers.get("Content-Type")
@@ -63,14 +66,17 @@ def splunkd_request(
                 timeout=timeout,
                 verify=verify,
             )
-        except Exception:
-            logging.error(msg_temp, splunkd_uri, "unknown", format_exc())
+        except Exception as e:
+            logging.error(msg_temp, splunkd_uri, "unknown", format_exc())  # deprecated
+            log_exception(logger, e, exc_label="unknown",
+                          msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown")
         else:
             if resp.status_code not in (200, 201):
                 if not (method == "GET" and resp.status_code == 404):
                     logging.debug(
                         msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp)
-                    )
+                    )  # deprecated
+                    logger.debug(msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp))
             else:
                 return resp
     else:

solnlib/splunk_rest_client.py

@@ -21,7 +21,6 @@
 calling splunklib SDK directly in business logic code.
 """
 
-import logging
 import os
 import sys
 import traceback
@@ -35,6 +34,9 @@
 from .net_utils import validate_scheme_host_port
 from .splunkenv import get_splunkd_access_info
 
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
 __all__ = ["SplunkRestClient"]
 MAX_REQUEST_RETRIES = 5
 
@@ -171,7 +173,7 @@ def request(url, message, **kwargs):
                 **kwargs,
             )
         except Exception:
-            logging.error(
+            logger.error(
                 "Failed to issue http request=%s to url=%s, error=%s",
                 method,
                 url,

solnlib/timer_queue.py

@@ -25,6 +25,10 @@
 
 import sortedcontainers as sc
 
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
+
 __all__ = ["Timer", "TimerQueueStruct", "TimerQueue"]
 
 
@@ -123,6 +127,9 @@ def remove_timer(self, timer: Timer):
         except ValueError:
             logging.info(
                 "Timer=%s is not in queue, move it to cancelling " "list", timer.ident
+            )  # deprecated
+            logger.info(
+                "Timer=%s is not in queue, move it to cancelling " "list", timer.ident
             )
         else:
             self._cancelling_timers[timer.ident] = timer
@@ -184,7 +191,8 @@ def check_and_execute(self) -> float:
             try:
                 timer()
             except Exception:
-                logging.error(traceback.format_exc())
+                logging.error(traceback.format_exc())  # deprecated
+                logger.error(traceback.format_exc())
 
         self.reset_timers(expired_timers)
         return _calc_sleep_time(next_expired_time)
@@ -231,7 +239,7 @@ def start(self):
         self._started = True
 
         self._thr.start()
-        logging.info("TimerQueue started.")
+        logger.info("TimerQueue started.")
 
     def stop(self):
         """Stop the timer queue."""
@@ -284,7 +292,8 @@ def _check_and_execute(self):
                     # Note, please make timer callback effective/short
                     timer()
                 except Exception:
-                    logging.error(traceback.format_exc())
+                    logging.error(traceback.format_exc())  # deprecated
+                    logger.error(traceback.format_exc())
 
             self._reset_timers(expired_timers)
 
@@ -295,7 +304,7 @@ def _check_and_execute(self):
                     break
             except Queue.Empty:
                 pass
-        logging.info("TimerQueue stopped.")
+        logger.info("TimerQueue stopped.")
 
     def _get_expired_timers(self):
         with self._lock:

solnlib/utils.py

@@ -17,7 +17,6 @@
 """Common utilities."""
 
 import datetime
-import logging
 import os
 import signal
 import time
@@ -26,6 +25,10 @@
 from typing import Any, Callable, List, Tuple, Union
 from urllib import parse as urlparse
 
+
+from solnlib.log import Logs
+logger = Logs().get_logger(__name__)
+
 __all__ = [
     "handle_teardown_signals",
     "datetime_to_seconds",
@@ -152,7 +155,7 @@ def wrapper(*args, **kwargs):
                 try:
                     return func(*args, **kwargs)
                 except Exception as e:
-                    logging.warning(
+                    logger.warning(
                         "Run function: %s failed: %s.",
                         func.__name__,
                         traceback.format_exc(),