feat: improve Application Insights logging and telemetry handling

documents/DeploymentGuide.md

@@ -274,6 +274,10 @@ azd auth login --tenant-id <tenant-id>
    > 3. Under the **Overview** section, locate the **Tenant ID** field. Copy the value displayed.
 
 ### 4.2 Start Deployment
+**NOTE:** If you are running the latest azd version (version 1.23.9), please run the following command. 
+```bash 
+azd config set provision.preflight off
+```
 
 ```shell
 azd up

infra/main.bicep

@@ -331,7 +331,6 @@ module applicationInsights 'br/public:avm/res/insights/component:0.7.1' = if (en
     flowType: 'Bluefield'
     // WAF aligned configuration for Monitoring
     workspaceResourceId: enableMonitoring ? logAnalyticsWorkspaceResourceId : ''
-    diagnosticSettings: enableMonitoring ? [{ workspaceResourceId: logAnalyticsWorkspaceResourceId }] : null
   }
 }
 // ========== Virtual Network and Networking Components ========== //

src/api/api/api_routes.py

@@ -12,28 +12,19 @@
 from services.chart_service import ChartService
 from common.logging.event_utils import track_event_if_configured
 from helpers.azure_credential_utils import get_azure_credential
-from azure.monitor.opentelemetry import configure_azure_monitor
+from auth.auth_utils import get_authenticated_user_details
 from opentelemetry import trace
 from opentelemetry.trace import Status, StatusCode
 
 router = APIRouter()
 
 logger = logging.getLogger(__name__)
 
-# Check if the Application Insights Instrumentation Key is set in the environment variables
-instrumentation_key = os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING")
-if instrumentation_key:
-    # Configure Application Insights if the Instrumentation Key is found
-    configure_azure_monitor(connection_string=instrumentation_key)
-    logging.info("Application Insights configured with the provided Instrumentation Key")
-else:
-    # Log a warning if the Instrumentation Key is not found
-    logging.warning("No Application Insights Instrumentation Key found. Skipping configuration")
-
 
 @router.get("/fetchChartData")
 async def fetch_chart_data():
     try:
+        logger.info("GET /fetchChartData called")
         chart_service = ChartService()
         response = await chart_service.fetch_chart_data()
         track_event_if_configured(
@@ -43,6 +34,10 @@ async def fetch_chart_data():
         return JSONResponse(content=response)
     except Exception as e:
         logger.exception("Error in fetch_chart_data: %s", str(e))
+        track_event_if_configured("FetchChartDataError", {
+            "error": str(e),
+            "error_type": type(e).__name__
+        })
         span = trace.get_current_span()
         if span is not None:
             span.record_exception(e)
@@ -53,7 +48,7 @@ async def fetch_chart_data():
 @router.post("/fetchChartDataWithFilters")
 async def fetch_chart_data_with_filters(chart_filters: ChartFilters):
     try:
-        logger.info(f"Received filters: {chart_filters}")
+        logger.info("Received filters: %s", chart_filters)
         chart_service = ChartService()
         response = await chart_service.fetch_chart_data_with_filters(chart_filters)
         track_event_if_configured(
@@ -69,6 +64,10 @@ async def fetch_chart_data_with_filters(chart_filters: ChartFilters):
         return JSONResponse(content=response)
     except Exception as e:
         logger.exception("Error in fetch_chart_data_with_filters: %s", str(e))
+        track_event_if_configured("FetchChartDataWithFiltersError", {
+            "error": str(e),
+            "error_type": type(e).__name__
+        })
         span = trace.get_current_span()
         if span is not None:
             span.record_exception(e)
@@ -79,6 +78,7 @@ async def fetch_chart_data_with_filters(chart_filters: ChartFilters):
 @router.get("/fetchFilterData")
 async def fetch_filter_data():
     try:
+        logger.info("GET /fetchFilterData called")
         chart_service = ChartService()
         response = await chart_service.fetch_filter_data()
         track_event_if_configured(
@@ -88,6 +88,10 @@ async def fetch_filter_data():
         return JSONResponse(content=response)
     except Exception as e:
         logger.exception("Error in fetch_filter_data: %s", str(e))
+        track_event_if_configured("FetchFilterDataError", {
+            "error": str(e),
+            "error_type": type(e).__name__
+        })
         span = trace.get_current_span()
         if span is not None:
             span.record_exception(e)
@@ -102,16 +106,42 @@ async def conversation(request: Request):
         request_json = await request.json()
         conversation_id = request_json.get("conversation_id")
         query = request_json.get("query")
+        authenticated_user = get_authenticated_user_details(request_headers=request.headers)
+        user_id = authenticated_user.get("user_principal_id", "")
+        logger.info("POST /chat called: conversation_id=%s, query_length=%d",
+                    conversation_id, len(query) if query else 0)
+
+        # Track chat request initiation
+        track_event_if_configured("ChatRequestReceived", {
+            "conversation_id": conversation_id,
+            "user_id": user_id
+        })
+
+        # Attach conversation_id to current span for Application Insights correlation
+        span = trace.get_current_span()
+        if span and conversation_id:
+            span.set_attribute("conversation_id", conversation_id)
+
         chat_service = ChatService()
-        result = await chat_service.stream_chat_request(conversation_id, query)
+        result = await chat_service.stream_chat_request(conversation_id, query, user_id=user_id)
+        logger.info("Chat stream initiated successfully for conversation_id=%s", conversation_id)
         track_event_if_configured(
             "ChatStreamSuccess",
-            {"conversation_id": conversation_id, "query": query}
+            {"conversation_id": conversation_id, "user_id": user_id, "query": query}
         )
         return StreamingResponse(result, media_type="application/json-lines")
 
     except Exception as ex:
         logger.exception("Error in conversation endpoint: %s", str(ex))
+
+        # Track specific error type
+        track_event_if_configured("ChatRequestError", {
+            "conversation_id": request_json.get("conversation_id") if 'request_json' in locals() else "",
+            "user_id": locals().get("user_id", ""),
+            "error": str(ex),
+            "error_type": type(ex).__name__
+        })
+
         span = trace.get_current_span()
         if span is not None:
             span.record_exception(ex)
@@ -121,6 +151,7 @@ async def conversation(request: Request):
 
 @router.get("/layout-config")
 async def get_layout_config():
+    logger.info("GET /layout-config called")
     layout_config_str = os.getenv("REACT_APP_LAYOUT_CONFIG", "")
     if layout_config_str:
         try:
@@ -129,6 +160,10 @@ async def get_layout_config():
             return JSONResponse(content=layout_config_json)    # Return the parsed JSON
         except json.JSONDecodeError as e:
             logger.exception("Failed to parse layout config JSON: %s", str(e))
+            track_event_if_configured("LayoutConfigParseError", {
+                "error": str(e),
+                "error_type": "JSONDecodeError"
+            })
             span = trace.get_current_span()
             if span is not None:
                 span.record_exception(e)
@@ -140,6 +175,7 @@ async def get_layout_config():
 
 @router.get("/display-chart-default")
 async def get_chart_config():
+    logger.info("GET /display-chart-default called")
     chart_config = os.getenv("DISPLAY_CHART_DEFAULT", "")
     if chart_config:
         track_event_if_configured("ChartDisplayDefaultFetched", {"value": chart_config})
@@ -158,6 +194,7 @@ async def fetch_azure_search_content_endpoint(request: Request):
         # Parse the request JSON
         request_json = await request.json()
         url = request_json.get("url")
+        logger.info("POST /fetch-azure-search-content called: url=%s", url)
 
         if not url:
             return JSONResponse(content={"error": "URL is required"}, status_code=400)
@@ -183,8 +220,10 @@ def fetch_content():
                     data = response.json()
                     content = data.get("content", "")
                     title = data.get("sourceurl", "")
+                    logger.info("Azure Search content fetched successfully: url=%s", url)
                     return {"content": content, "title": title}
                 else:
+                    logger.warning("Azure Search content fetch failed: url=%s, status=%d", url, response.status_code)
                     return {"error": f"HTTP {response.status_code}"}
             except Exception:
                 logger.exception("Exception occurred while making the HTTP request")