Merge pull request #59 from microsoft/PSL-US-26761

Roopan-Microsoft · web-flow · commit 301fab714db1 · 2025-11-28T19:48:38.000+05:30
feat: Implementation of Configurable Logging Control via Flag
diff --git a/infra/main.bicep b/infra/main.bicep
@@ -856,6 +856,14 @@ module appConfiguration 'br/public:avm/res/app-configuration/configuration-store
         name: 'APP_LOGGING_LEVEL'
         value: 'INFO'
       }
+      {
+        name: 'AZURE_PACKAGE_LOGGING_LEVEL'
+        value: 'INFO'
+      }
+      {
+        name: 'AZURE_LOGGING_PACKAGES'
+        value: ''
+      }
       {
         name: 'AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME'
         value: ''
diff --git a/infra/main_custom.bicep b/infra/main_custom.bicep
@@ -859,6 +859,14 @@ module appConfiguration 'br/public:avm/res/app-configuration/configuration-store
         name: 'APP_LOGGING_LEVEL'
         value: 'INFO'
       }
+      {
+        name: 'AZURE_PACKAGE_LOGGING_LEVEL'
+        value: 'INFO'
+      }
+      {
+        name: 'AZURE_LOGGING_PACKAGES'
+        value: ''
+      }
       {
         name: 'AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME'
         value: ''
diff --git a/src/backend-api/src/app/.env.example b/src/backend-api/src/app/.env.example
@@ -3,3 +3,7 @@ APP_CONFIGURATION_URL=""
 # APP_LOGGING_ENABLE=true
 # APP_LOGGING_LEVEL="INFO"
 
+# Azure Logging Configuration
+# AZURE_PACKAGE_LOGGING_LEVEL="WARNING"  # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL
+# AZURE_LOGGING_PACKAGES="azure.core.pipeline.policies.http_logging_policy,azure.storage.blob,azure.storage.queue,azure.core,azure.identity,azure.storage,azure.core.pipeline,azure.core.pipeline.policies,azure.core.pipeline.transport,openai,openai._client,httpx,httpcore,semantic_kernel,urllib3,msal"
+
diff --git a/src/backend-api/src/app/libs/application/application_configuration.py b/src/backend-api/src/app/libs/application/application_configuration.py
@@ -33,6 +33,10 @@ class Configuration(_configuration_base, KernelBaseSettings):
     app_logging_level: str = Field(default="INFO")
     app_sample_variable: str = Field(default="Hello World!")
 
+    # Azure logging configuration
+    azure_package_logging_level: str = Field(default="WARNING", alias="AZURE_PACKAGE_LOGGING_LEVEL")
+    azure_logging_packages: str | None = Field(default=None, alias="AZURE_LOGGING_PACKAGES")
+
     global_llm_service: str | None = "AzureOpenAI"
     cosmos_db_process_log_container: str | None = Field(
         default=None, env="COSMOS_DB_PROCESS_LOG_CONTAINER"
diff --git a/src/backend-api/src/app/libs/base/application_base.py b/src/backend-api/src/app/libs/base/application_base.py
@@ -60,6 +60,12 @@ def __init__(self, env_file_path: str | None = None, **data):
             )
             logging.basicConfig(level=logging_level)
 
+            # Configure Azure package logging levels only if packages are specified
+            if self.application_context.configuration.azure_logging_packages:
+                azure_level = getattr(logging, self.application_context.configuration.azure_package_logging_level.upper(), logging.WARNING)
+                for logger_name in filter(None, (pkg.strip() for pkg in self.application_context.configuration.azure_logging_packages.split(','))):
+                    logging.getLogger(logger_name).setLevel(azure_level)
+
         # Initialize the application
         self.initialize()
 
diff --git a/src/backend-api/src/app/routers/router_debug.py b/src/backend-api/src/app/routers/router_debug.py
@@ -19,6 +19,8 @@ async def get_config_debug(request: Request):
     config_dict = {
         "app_logging_enable": config.app_logging_enable,
         "app_logging_level": config.app_logging_level,
+        "azure_package_logging_level": config.azure_package_logging_level,
+        "azure_logging_packages": config.azure_logging_packages,
         "cosmos_db_account_url": config.cosmos_db_account_url,
         "cosmos_db_database_name": config.cosmos_db_database_name,
         "cosmos_db_process_container": config.cosmos_db_process_container,
diff --git a/src/backend-api/src/tests/application/test_app_configuration.py b/src/backend-api/src/tests/application/test_app_configuration.py
@@ -28,6 +28,14 @@ def test_configuration_fields():
     assert isinstance(config.app_logging_enable, bool)
     assert config.app_logging_enable is True
 
+    assert hasattr(config, "azure_package_logging_level")
+    assert isinstance(config.azure_package_logging_level, str)
+    assert config.azure_package_logging_level == "INFO"
+
+    assert hasattr(config, "azure_logging_packages")
+    assert isinstance(config.azure_logging_packages, list)
+    assert config.azure_logging_packages == []
+
     assert hasattr(config, "app_logging_level")
     assert isinstance(config.app_logging_level, str)
     assert config.app_logging_level == "INFO"
diff --git a/src/backend-api/test_config.py b/src/backend-api/test_config.py
@@ -20,6 +20,8 @@ def test_configuration():
     config_values = {
         "app_logging_enable": config.app_logging_enable,
         "app_logging_level": config.app_logging_level,
+        "azure_package_logging_level": config.azure_package_logging_level,
+        "azure_logging_packages": config.azure_logging_packages,
         "cosmos_db_account_url": config.cosmos_db_account_url,
         "cosmos_db_database_name": config.cosmos_db_database_name,
         "cosmos_db_process_container": config.cosmos_db_process_container,
diff --git a/src/processor/.env.example b/src/processor/.env.example
@@ -0,0 +1,19 @@
+APP_CONFIGURATION_URL=""
+
+# Application Logging Configuration
+# APP_LOGGING_ENABLE=true
+# APP_LOGGING_LEVEL="INFO"
+
+# Azure Logging Configuration
+# AZURE_PACKAGE_LOGGING_LEVEL="WARNING"  # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL
+# AZURE_LOGGING_PACKAGES="azure.core.pipeline.policies.http_logging_policy,azure.storage.blob,azure.storage.queue,azure.core,azure.identity,azure.storage,azure.core.pipeline,azure.core.pipeline.policies,azure.core.pipeline.transport,openai,openai._client,httpx,httpcore,semantic_kernel,urllib3,msal"
+
+# Database Configuration
+# COSMOS_DB_ACCOUNT_URL="http://<cosmos url>"
+# COSMOS_DB_DATABASE_NAME="<database name>"
+# COSMOS_DB_CONTAINER_NAME="<container name>"
+
+# Storage Configuration
+# STORAGE_QUEUE_ACCOUNT="http://<storage queue url>"
+# STORAGE_ACCOUNT_PROCESS_QUEUE="http://<storage account process queue url>"
+# STORAGE_QUEUE_NAME="processes-queue"
diff --git a/src/processor/src/libs/application/application_configuration.py b/src/processor/src/libs/application/application_configuration.py
@@ -29,6 +29,10 @@ class Configuration(_configuration_base):
     # api_key: str
     app_logging_enable: bool = Field(default=False, alias="APP_LOGGING_ENABLE")
     app_logging_level: str = Field(default="INFO", alias="APP_LOGGING_LEVEL")
+
+    # Azure logging configuration
+    azure_package_logging_level: str = Field(default="WARNING", alias="AZURE_PACKAGE_LOGGING_LEVEL")
+    azure_logging_packages: str | None = Field(default=None, alias="AZURE_LOGGING_PACKAGES")
     cosmos_db_account_url: str = Field(
         default="http://<cosmos url>", alias="COSMOS_DB_ACCOUNT_URL"
     )
diff --git a/src/processor/src/libs/base/ApplicationBase.py b/src/processor/src/libs/base/ApplicationBase.py
@@ -77,6 +77,12 @@ async def initialize_async(self):
             # Ensure non-debug mode suppresses all debug messages
             logging.basicConfig(level=logging.WARNING)
 
+        # Configure Azure package logging levels only if packages are specified
+        if self.app_context.configuration.azure_logging_packages:
+            azure_level = getattr(logging, self.app_context.configuration.azure_package_logging_level.upper(), logging.WARNING)
+            for logger_name in filter(None, (pkg.strip() for pkg in self.app_context.configuration.azure_logging_packages.split(','))):
+                logging.getLogger(logger_name).setLevel(azure_level)
+
         # Always suppress semantic kernel debug messages unless explicitly in debug mode
         if not self.debug_mode:
             logging.getLogger("semantic_kernel").setLevel(logging.WARNING)
diff --git a/src/processor/src/utils/logging_utils.py b/src/processor/src/utils/logging_utils.py
@@ -18,21 +18,26 @@
 from semantic_kernel.exceptions import ServiceException
 
 
-def configure_application_logging(debug_mode: bool = False):
+def configure_application_logging(debug_mode: bool = False, config=None):
     """
     Comprehensive logging configuration with third-party suppression.
 
     Replaces suppress_verbose_logging() from quiet_logging.py
 
     Args:
         debug_mode: If True, allows some debug logging. If False, suppresses all debug output.
+        config: Configuration object with logging settings. If None, uses default settings.
     """
     # Set root logger level
     if debug_mode:
         logging.basicConfig(level=logging.DEBUG)
         print("🐛 Debug logging enabled")
     else:
-        logging.basicConfig(level=logging.INFO)
+        # Use configuration if available, otherwise default to INFO
+        basic_level = logging.INFO
+        if config and hasattr(config, 'app_logging_level'):
+            basic_level = getattr(logging, config.app_logging_level.upper(), logging.INFO)
+        logging.basicConfig(level=basic_level)
 
     # Comprehensive list of verbose loggers to suppress
     verbose_loggers = [
@@ -83,33 +88,43 @@ def configure_application_logging(debug_mode: bool = False):
         "charset_normalizer",
     ]
 
-    # Set levels for all verbose loggers
-    for logger_name in verbose_loggers:
-        logger = logging.getLogger(logger_name)
-        if debug_mode:
-            # In debug mode, still reduce verbosity to INFO for most, WARNING for HTTP
-            if any(
-                http_term in logger_name.lower()
-                for http_term in ["http", "client", "connection", "pipeline"]
-            ):
-                logger.setLevel(logging.WARNING)
+    # Configure Azure package logging levels from configuration only if packages are specified
+    if config and hasattr(config, 'azure_logging_packages') and hasattr(config, 'azure_package_logging_level') and config.azure_logging_packages:
+        # Use configuration-based approach
+        azure_level = getattr(logging, config.azure_package_logging_level.upper(), logging.WARNING)
+        package_list = [pkg.strip() for pkg in config.azure_logging_packages.split(',')]
+        for logger_name in package_list:
+            if logger_name:  # Skip empty strings
+                logging.getLogger(logger_name).setLevel(azure_level)
+    else:
+        # Fallback to existing logic for backward compatibility
+        # Set levels for all verbose loggers
+        for logger_name in verbose_loggers:
+            logger = logging.getLogger(logger_name)
+            if debug_mode:
+                # In debug mode, still reduce verbosity to INFO for most, WARNING for HTTP
+                if any(
+                    http_term in logger_name.lower()
+                    for http_term in ["http", "client", "connection", "pipeline"]
+                ):
+                    logger.setLevel(logging.WARNING)
+                else:
+                    logger.setLevel(logging.INFO)
             else:
-                logger.setLevel(logging.INFO)
-        else:
-            # In production, suppress to WARNING for all
-            logger.setLevel(logging.WARNING)
-
-    # Special cases: These are ALWAYS set to WARNING due to extreme verbosity
-    always_warning_loggers = [
-        "azure.core.pipeline.policies.http_logging_policy",
-        "httpx",
-        "httpcore",
-        "openai._client",
-        "urllib3.connectionpool",
-    ]
+                # In production, suppress to WARNING for all
+                logger.setLevel(logging.WARNING)
 
-    for logger_name in always_warning_loggers:
-        logging.getLogger(logger_name).setLevel(logging.WARNING)
+        # Special cases: These are ALWAYS set to WARNING due to extreme verbosity
+        always_warning_loggers = [
+            "azure.core.pipeline.policies.http_logging_policy",
+            "httpx",
+            "httpcore",
+            "openai._client",
+            "urllib3.connectionpool",
+        ]
+
+        for logger_name in always_warning_loggers:
+            logging.getLogger(logger_name).setLevel(logging.WARNING)
 
     # Set environment variables to suppress verbose output at the source
     os.environ.setdefault("HTTPX_LOG_LEVEL", "WARNING")
