Address PR #586 review comments

- Change emit_all() logging from INFO to DEBUG to reduce production log volume
- Optional token-detail fields default to None when absent instead of 0
- Remove redundant extract_usage_from_stream_chunk() fallback in TokenUsageScope.add()
- Pin azure-monitor-events-extension to ==0.1.0 in both projects
- Revert enableMonitoring to false in default parameters

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Prachig-Microsoft

infra/main.parameters.json

@@ -36,7 +36,7 @@
       "value": "${AZURE_ENV_IMAGETAG=latest_v2}"
     },
     "enableMonitoring": {
-      "value": true
+      "value": false
     }
   }
 }

src/ContentProcessor/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
     "azure-ai-inference==1.0.0b9",
     "azure-appconfiguration==1.8.0",
     "azure-identity==1.26.0b1",
-    "azure-monitor-events-extension>=0.1.0",
+    "azure-monitor-events-extension==0.1.0",
     "azure-monitor-opentelemetry==1.8.7",
     "azure-storage-blob==12.29.0b1",
     "azure-storage-queue==12.16.0b1",

src/ContentProcessor/src/libs/llm_token_telemetry.py

@@ -326,11 +326,11 @@ def extract_realtime_usage(response_obj: Any) -> Optional[TokenUsage]:
         input_tokens=inp,
         output_tokens=out,
         total_tokens=tot,
-        input_audio_tokens=_to_int(_get(in_details, "audio_tokens")),
-        input_text_tokens=_to_int(_get(in_details, "text_tokens")),
-        input_cached_tokens=_to_int(_get(in_details, "cached_tokens")),
-        output_audio_tokens=_to_int(_get(out_details, "audio_tokens")),
-        output_text_tokens=_to_int(_get(out_details, "text_tokens")),
+        input_audio_tokens=_to_int(_get(in_details, "audio_tokens")) if _get(in_details, "audio_tokens") is not None else None,
+        input_text_tokens=_to_int(_get(in_details, "text_tokens")) if _get(in_details, "text_tokens") is not None else None,
+        input_cached_tokens=_to_int(_get(in_details, "cached_tokens")) if _get(in_details, "cached_tokens") is not None else None,
+        output_audio_tokens=_to_int(_get(out_details, "audio_tokens")) if _get(out_details, "audio_tokens") is not None else None,
+        output_text_tokens=_to_int(_get(out_details, "text_tokens")) if _get(out_details, "text_tokens") is not None else None,
     )
     if record.has_any or any(
         v for v in (
@@ -776,7 +776,7 @@ def emit_all(
             **dimensions,
         )
 
-        self._log.info(
+        self._log.debug(
             "[TOKEN USAGE] agent=%s model=%s input=%d output=%d total=%d %s",
             agent_name,
             model_deployment_name,
@@ -856,7 +856,7 @@ def add(self, source: Any) -> Optional[TokenUsage]:
         """
         start_ns = time.perf_counter_ns()
         try:
-            found = extract_usage(source) or extract_usage_from_stream_chunk(source)
+            found = extract_usage(source)
         except Exception as exc:
             logger.debug("TokenUsageScope.add failed: %s", exc, exc_info=True)
             return None

src/ContentProcessorWorkflow/pyproject.toml

@@ -14,7 +14,7 @@ dependencies = [
     "azure-appconfiguration==1.8.0",
     "azure-core==1.38.0",
     "azure-identity==1.26.0b1",
-    "azure-monitor-events-extension>=0.1.0",
+    "azure-monitor-events-extension==0.1.0",
     "azure-monitor-opentelemetry==1.8.7",
     "azure-storage-blob==12.29.0b1",
     "azure-storage-file-datalake==12.23.0",

src/ContentProcessorWorkflow/src/libs/llm_token_telemetry.py

@@ -326,11 +326,11 @@ def extract_realtime_usage(response_obj: Any) -> Optional[TokenUsage]:
         input_tokens=inp,
         output_tokens=out,
         total_tokens=tot,
-        input_audio_tokens=_to_int(_get(in_details, "audio_tokens")),
-        input_text_tokens=_to_int(_get(in_details, "text_tokens")),
-        input_cached_tokens=_to_int(_get(in_details, "cached_tokens")),
-        output_audio_tokens=_to_int(_get(out_details, "audio_tokens")),
-        output_text_tokens=_to_int(_get(out_details, "text_tokens")),
+        input_audio_tokens=_to_int(_get(in_details, "audio_tokens")) if _get(in_details, "audio_tokens") is not None else None,
+        input_text_tokens=_to_int(_get(in_details, "text_tokens")) if _get(in_details, "text_tokens") is not None else None,
+        input_cached_tokens=_to_int(_get(in_details, "cached_tokens")) if _get(in_details, "cached_tokens") is not None else None,
+        output_audio_tokens=_to_int(_get(out_details, "audio_tokens")) if _get(out_details, "audio_tokens") is not None else None,
+        output_text_tokens=_to_int(_get(out_details, "text_tokens")) if _get(out_details, "text_tokens") is not None else None,
     )
     if record.has_any or any(
         v for v in (
@@ -776,7 +776,7 @@ def emit_all(
             **dimensions,
         )
 
-        self._log.info(
+        self._log.debug(
             "[TOKEN USAGE] agent=%s model=%s input=%d output=%d total=%d %s",
             agent_name,
             model_deployment_name,
@@ -856,7 +856,7 @@ def add(self, source: Any) -> Optional[TokenUsage]:
         """
         start_ns = time.perf_counter_ns()
         try:
-            found = extract_usage(source) or extract_usage_from_stream_chunk(source)
+            found = extract_usage(source)
         except Exception as exc:
             logger.debug("TokenUsageScope.add failed: %s", exc, exc_info=True)
             return None