fix: dev to main - Private Networking for Web Apps, Improve CI Test Reporting, and Strengthen Error Handling

Author: Avijit-Microsoft

.github/workflows/test.yml

@@ -31,6 +31,7 @@ on:
 permissions:
   contents: read
   actions: read
+  pull-requests: write
 
 jobs:
   #   frontend_tests:
@@ -106,7 +107,19 @@ jobs:
       - name: Run Backend Tests with Coverage
         if: env.skip_backend_tests == 'false'
         run: |
-          pytest --cov=. --cov-report=term-missing --cov-report=xml ./src/tests/api
+          pytest --cov=. --cov-report=term-missing --cov-report=xml --junitxml=pytest.xml ./src/tests/api
+
+      - name: Pytest Coverage Comment
+        if: |
+          always() &&
+          github.event_name == 'pull_request' &&
+          github.event.pull_request.head.repo.fork == false &&
+          env.skip_backend_tests == 'false'
+        uses: MishaKav/pytest-coverage-comment@26f986d2599c288bb62f623d29c2da98609e9cd4  # v1.6.0
+        with:
+          pytest-xml-coverage-path: coverage.xml
+          junitxml-path: pytest.xml
+          report-only-changed-files: true
 
       - name: Skip Backend Tests
         if: env.skip_backend_tests == 'true'

azure.yaml

@@ -8,7 +8,6 @@ name: conversation-knowledge-mining
 
 requiredVersions:
   azd: ">= 1.18.0 != 1.23.9"
-  bicep: ">= 0.33.0"
 
 metadata:
   template: conversation-knowledge-mining@1.0

docs/workshop/docs/workshop/Challenge-5/python/utility.py

@@ -242,7 +242,7 @@ def schema_to_tool(schema: Any):
                 return json.loads(
                     assistant_message.tool_calls[0].function.arguments, strict=False
                 )
-            except:
+            except Exception:
                 return assistant_message.tool_calls[0].function.arguments
 
     def get_structured_output_answer(
@@ -348,7 +348,6 @@ def generate_scenes(
         scene_generation_prompt = Template(SCENE_GENERATION_PROMPT).substitute(
             descriptions=next_segment_content
         )
-        scence_response = VideoSceneResponse(scenes=[])
         scence_response = openai_assistant.get_structured_output_answer(
             "", scene_generation_prompt, VideoSceneResponse
         )
@@ -433,7 +432,6 @@ def generate_chapters(
     chapter_generation_prompt = Template(CHAPTER_GENERATION_PROMPT).substitute(
         descriptions=scene_descriptions
     )
-    chapter_response = VideoChapterResponse(chapters=[])
     chapter_response = openai_assistant.get_structured_output_answer(
         "", chapter_generation_prompt, VideoChapterResponse
     )
@@ -460,7 +458,6 @@ def aggregate_tags(
     tags_dedup = set(map(lambda x: re.sub(r'^ ', '', x), tags))
     tag_dedup_prompt = Template(DEDUP_PROMPT).substitute(tag_list=tags_dedup)
 
-    tag_response = VideoTagResponse(tags=[])
     tag_response = openai_assistant.get_structured_output_answer(
         "", tag_dedup_prompt, VideoTagResponse
     )

infra/main.bicep

@@ -445,6 +445,7 @@ var privateDnsZones = [
   'privatelink.documents.azure.com'
   'privatelink${environment().suffixes.sqlServerHostname}'
   'privatelink.search.windows.net'
+  'privatelink.azurewebsites.net'
 ]
 
 // DNS Zone Index Constants
@@ -459,6 +460,7 @@ var dnsZoneIndex = {
   cosmosDB: 7
   sqlServer: 8
   search: 9
+  webApp: 10
 }
 
 // ===================================================
@@ -1361,7 +1363,22 @@ module webSiteBackend 'modules/web-sites.bicep' = {
     vnetRouteAllEnabled: enablePrivateNetworking ? true : false
     vnetImagePullEnabled: enablePrivateNetworking ? true : false
     virtualNetworkSubnetId: enablePrivateNetworking ? virtualNetwork!.outputs.webSubnetResourceId : null
-    publicNetworkAccess: 'Enabled'
+    publicNetworkAccess: enablePrivateNetworking ? 'Disabled' : 'Enabled'
+    privateEndpoints: enablePrivateNetworking
+      ? [
+          {
+            name: 'pep-${backendWebSiteResourceName}'
+            customNetworkInterfaceName: 'nic-${backendWebSiteResourceName}'
+            privateDnsZoneGroup: {
+              privateDnsZoneGroupConfigs: [
+                { privateDnsZoneResourceId: avmPrivateDnsZones[dnsZoneIndex.webApp]!.outputs.resourceId }
+              ]
+            }
+            service: 'sites'
+            subnetResourceId: virtualNetwork!.outputs.pepsSubnetResourceId
+          }
+        ]
+      : []
   }
 }
 
@@ -1388,7 +1405,8 @@ module webSiteFrontend 'modules/web-sites.bicep' = {
       {
         name: 'appsettings'
         properties: {
-          APP_API_BASE_URL: 'https://api-${solutionSuffix}.azurewebsites.net'
+          APP_API_BASE_URL: enablePrivateNetworking ? '' : 'https://api-${solutionSuffix}.azurewebsites.net'
+          BACKEND_API_HOST: enablePrivateNetworking ? 'api-${solutionSuffix}.azurewebsites.net' : ''
         }
         applicationInsightResourceId: enableMonitoring ? applicationInsights!.outputs.resourceId : null
       }

infra/main.json

@@ -6,7 +6,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.41.2.15936",
-      "templateHash": "949305639975172329"
+      "templateHash": "18133262101573574086"
     }
   },
   "parameters": {
@@ -408,7 +408,7 @@
     "sqlServerResourceName": "[format('sql-{0}', variables('solutionSuffix'))]",
     "sqlDbModuleName": "[format('sqldb-{0}', variables('solutionSuffix'))]",
     "webServerFarmResourceName": "[format('asp-{0}', variables('solutionSuffix'))]",
-    "reactAppLayoutConfig": "{\r\n  \"appConfig\": {\r\n    \"THREE_COLUMN\": {\r\n      \"DASHBOARD\": 50,\r\n      \"CHAT\": 33,\r\n      \"CHATHISTORY\": 17\r\n    },\r\n    \"TWO_COLUMN\": {\r\n      \"DASHBOARD_CHAT\": {\r\n        \"DASHBOARD\": 65,\r\n        \"CHAT\": 35\r\n      },\r\n      \"CHAT_CHATHISTORY\": {\r\n        \"CHAT\": 80,\r\n        \"CHATHISTORY\": 20\r\n      }\r\n    }\r\n  },\r\n  \"charts\": [\r\n    {\r\n      \"id\": \"SATISFIED\",\r\n      \"name\": \"Satisfied\",\r\n      \"type\": \"card\",\r\n      \"layout\": { \"row\": 1, \"column\": 1, \"height\": 11 }\r\n    },\r\n    {\r\n      \"id\": \"TOTAL_CALLS\",\r\n      \"name\": \"Total Calls\",\r\n      \"type\": \"card\",\r\n      \"layout\": { \"row\": 1, \"column\": 2, \"span\": 1 }\r\n    },\r\n    {\r\n      \"id\": \"AVG_HANDLING_TIME\",\r\n      \"name\": \"Average Handling Time\",\r\n      \"type\": \"card\",\r\n      \"layout\": { \"row\": 1, \"column\": 3, \"span\": 1 }\r\n    },\r\n    {\r\n      \"id\": \"SENTIMENT\",\r\n      \"name\": \"Topics Overview\",\r\n      \"type\": \"donutchart\",\r\n      \"layout\": { \"row\": 2, \"column\": 1, \"width\": 40, \"height\": 44.5 }\r\n    },\r\n    {\r\n      \"id\": \"AVG_HANDLING_TIME_BY_TOPIC\",\r\n      \"name\": \"Average Handling Time By Topic\",\r\n      \"type\": \"bar\",\r\n      \"layout\": { \"row\": 2, \"column\": 2, \"row-span\": 2, \"width\": 60 }\r\n    },\r\n    {\r\n      \"id\": \"TOPICS\",\r\n      \"name\": \"Trending Topics\",\r\n      \"type\": \"table\",\r\n      \"layout\": { \"row\": 3, \"column\": 1, \"span\": 2 }\r\n    },\r\n    {\r\n      \"id\": \"KEY_PHRASES\",\r\n      \"name\": \"Key Phrases\",\r\n      \"type\": \"wordcloud\",\r\n      \"layout\": { \"row\": 3, \"column\": 2, \"height\": 44.5 }\r\n    }\r\n  ]\r\n}",
+    "reactAppLayoutConfig": "{\n  \"appConfig\": {\n    \"THREE_COLUMN\": {\n      \"DASHBOARD\": 50,\n      \"CHAT\": 33,\n      \"CHATHISTORY\": 17\n    },\n    \"TWO_COLUMN\": {\n      \"DASHBOARD_CHAT\": {\n        \"DASHBOARD\": 65,\n        \"CHAT\": 35\n      },\n      \"CHAT_CHATHISTORY\": {\n        \"CHAT\": 80,\n        \"CHATHISTORY\": 20\n      }\n    }\n  },\n  \"charts\": [\n    {\n      \"id\": \"SATISFIED\",\n      \"name\": \"Satisfied\",\n      \"type\": \"card\",\n      \"layout\": { \"row\": 1, \"column\": 1, \"height\": 11 }\n    },\n    {\n      \"id\": \"TOTAL_CALLS\",\n      \"name\": \"Total Calls\",\n      \"type\": \"card\",\n      \"layout\": { \"row\": 1, \"column\": 2, \"span\": 1 }\n    },\n    {\n      \"id\": \"AVG_HANDLING_TIME\",\n      \"name\": \"Average Handling Time\",\n      \"type\": \"card\",\n      \"layout\": { \"row\": 1, \"column\": 3, \"span\": 1 }\n    },\n    {\n      \"id\": \"SENTIMENT\",\n      \"name\": \"Topics Overview\",\n      \"type\": \"donutchart\",\n      \"layout\": { \"row\": 2, \"column\": 1, \"width\": 40, \"height\": 44.5 }\n    },\n    {\n      \"id\": \"AVG_HANDLING_TIME_BY_TOPIC\",\n      \"name\": \"Average Handling Time By Topic\",\n      \"type\": \"bar\",\n      \"layout\": { \"row\": 2, \"column\": 2, \"row-span\": 2, \"width\": 60 }\n    },\n    {\n      \"id\": \"TOPICS\",\n      \"name\": \"Trending Topics\",\n      \"type\": \"table\",\n      \"layout\": { \"row\": 3, \"column\": 1, \"span\": 2 }\n    },\n    {\n      \"id\": \"KEY_PHRASES\",\n      \"name\": \"Key Phrases\",\n      \"type\": \"wordcloud\",\n      \"layout\": { \"row\": 3, \"column\": 2, \"height\": 44.5 }\n    }\n  ]\n}",
     "backendWebSiteResourceName": "[format('api-{0}', variables('solutionSuffix'))]",
     "webSiteResourceName": "[format('app-{0}', variables('solutionSuffix'))]"
   },
@@ -28296,9 +28296,9 @@
       },
       "dependsOn": [
         "aiFoundryAiServices",
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').cognitiveServices)]",
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').openAI)]",
         "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').aiServices)]",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').openAI)]",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').cognitiveServices)]",
         "virtualNetwork"
       ]
     },
@@ -41907,10 +41907,10 @@
         }
       },
       "dependsOn": [
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageFile)]",
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageQueue)]",
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageDfs)]",
         "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageBlob)]",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageDfs)]",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageQueue)]",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageFile)]",
         "userAssignedIdentity",
         "virtualNetwork"
       ]

infra/main_custom.bicep

@@ -446,6 +446,7 @@ var privateDnsZones = [
   'privatelink.documents.azure.com'
   'privatelink${environment().suffixes.sqlServerHostname}'
   'privatelink.search.windows.net'
+  'privatelink.azurewebsites.net'
 ]
 
 // DNS Zone Index Constants
@@ -460,6 +461,7 @@ var dnsZoneIndex = {
   cosmosDB: 7
   sqlServer: 8
   search: 9
+  webApp: 10
 }
 
 // ===================================================
@@ -1363,7 +1365,22 @@ module webSiteBackend 'modules/web-sites.bicep' = {
     vnetRouteAllEnabled: enablePrivateNetworking ? true : false
     vnetImagePullEnabled: enablePrivateNetworking ? true : false
     virtualNetworkSubnetId: enablePrivateNetworking ? virtualNetwork!.outputs.webSubnetResourceId : null
-    publicNetworkAccess: 'Enabled'
+    publicNetworkAccess: enablePrivateNetworking ? 'Disabled' : 'Enabled'
+    privateEndpoints: enablePrivateNetworking
+      ? [
+          {
+            name: 'pep-${backendWebSiteResourceName}'
+            customNetworkInterfaceName: 'nic-${backendWebSiteResourceName}'
+            privateDnsZoneGroup: {
+              privateDnsZoneGroupConfigs: [
+                { privateDnsZoneResourceId: avmPrivateDnsZones[dnsZoneIndex.webApp]!.outputs.resourceId }
+              ]
+            }
+            service: 'sites'
+            subnetResourceId: virtualNetwork!.outputs.pepsSubnetResourceId
+          }
+        ]
+      : []
   }
 }
 
@@ -1391,9 +1408,10 @@ module webSiteFrontend 'modules/web-sites.bicep' = {
         properties: {
           SCM_DO_BUILD_DURING_DEPLOYMENT: 'true'
           ENABLE_ORYX_BUILD: 'true'
-          REACT_APP_API_BASE_URL: 'https://api-${solutionSuffix}.azurewebsites.net'
+          REACT_APP_API_BASE_URL: enablePrivateNetworking ? '' : 'https://api-${solutionSuffix}.azurewebsites.net'
           WEBSITE_NODE_DEFAULT_VERSION: '~20'
-          APP_API_BASE_URL: 'https://api-${solutionSuffix}.azurewebsites.net'
+          APP_API_BASE_URL: enablePrivateNetworking ? '' : 'https://api-${solutionSuffix}.azurewebsites.net'
+          BACKEND_API_HOST: enablePrivateNetworking ? 'api-${solutionSuffix}.azurewebsites.net' : ''
         }
         applicationInsightResourceId: enableMonitoring ? applicationInsights!.outputs.resourceId : null
       }

infra/scripts/fabric_scripts/create_fabric_items.py

@@ -1,8 +1,6 @@
-from azure.identity import ManagedIdentityCredential
 import base64
 import json
 import requests
-import pandas as pd
 import os
 from glob import iglob
 import zipfile
@@ -98,11 +96,11 @@
         # upload extracted folder
         file_names = [f for f in iglob(os.path.join(local_path, "**", "*"), recursive=True) if os.path.isfile(f)]
         # print('file_names ex', file_names)
-        for file_name in file_names:
-            upload_file_name = os.path.basename(file_name)
+        for extracted_file in file_names:
+            upload_file_name = os.path.basename(extracted_file)
             file_client = directory_client.get_file_client("cu_audio_files_all/" + upload_file_name)
-            # with open(file=os.path.join(extract_dir, file_name), mode="rb") as data:
-            with open(file=file_name, mode="rb") as data:
+            # with open(file=os.path.join(extract_dir, extracted_file), mode="rb") as data:
+            with open(file=extracted_file, mode="rb") as data:
                 # print('data', data)
                 file_client.upload_data(data, overwrite=True)
 
@@ -127,7 +125,7 @@
   env_res = requests.get(fabric_env_url, headers=fabric_headers)
   env_res_id = env_res.json()['value'][0]['id']
   # print(env_res.json())
-except:
+except Exception:  # Environments may not be provisioned yet
   env_res_id = ''
 
 #create notebook items
@@ -150,14 +148,14 @@
         notebook_json['metadata']['dependencies']['lakehouse']['default_lakehouse'] = lakehouse_res.json()['id']
         notebook_json['metadata']['dependencies']['lakehouse']['default_lakehouse_name'] = lakehouse_res.json()['displayName']
         notebook_json['metadata']['dependencies']['lakehouse']['default_lakehouse_workspace_id'] = lakehouse_res.json()['workspaceId']
-    except:
+    except Exception:  # Lakehouse metadata may not be available
         pass
 
     if env_res_id != '':
         try:
             notebook_json['metadata']['dependencies']['environment']['environmentId'] = env_res_id
             notebook_json['metadata']['dependencies']['environment']['workspaceId'] = lakehouse_res.json()['workspaceId']
-        except:
+        except Exception:  # Environment metadata may not be available
             pass
 
 
@@ -178,8 +176,7 @@
         }
     }
 
-    fabric_response = requests.post(fabric_items_url, headers=fabric_headers, json=notebook_data)
-    #print(fabric_response.json())
+    requests.post(fabric_items_url, headers=fabric_headers, json=notebook_data)
 
 time.sleep(120)
 

infra/scripts/index_scripts/02_create_cu_template_audio.py

@@ -36,7 +36,7 @@
     analyzer = client.get_analyzer_detail_by_id(ANALYZER_ID)
     if analyzer is not None:
         client.delete_analyzer(ANALYZER_ID)
-except Exception:
+except Exception:  # Analyzer may not exist yet, safe to ignore
     pass
 
 response = client.begin_create_analyzer(ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE_FILE)

infra/scripts/index_scripts/02_create_cu_template_text.py

@@ -31,7 +31,7 @@
     analyzer = client.get_analyzer_detail_by_id(ANALYZER_ID)
     if analyzer is not None:
         client.delete_analyzer(ANALYZER_ID)
-except Exception:
+except Exception:  # Analyzer may not exist yet, safe to ignore
     pass
 
 response = client.begin_create_analyzer(ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE_FILE)

infra/scripts/index_scripts/03_cu_process_data_text.py

@@ -381,10 +381,10 @@ async def process_files():
 
                 docs.extend(await prepare_search_doc(content, conversation_id, path.name, embeddings_client))
                 counter += 1
-            except Exception:
+            except Exception:  # Skip files that fail processing
                 pass
             if docs != [] and counter % 10 == 0:
-                result = search_client.upload_documents(documents=docs)
+                search_client.upload_documents(documents=docs)
                 docs = []
         if docs:
             search_client.upload_documents(documents=docs)
@@ -533,7 +533,6 @@ async def call_topic_mining_agent(topics_str1):
     column_names = [i[0] for i in cursor.description]
     df_topics = pd.DataFrame(rows, columns=column_names)
     mined_topics_list = df_topics['label'].tolist()
-    mined_topics = ", ".join(mined_topics_list)
     print(f"✓ Mined {len(mined_topics_list)} topics")
 
     async def call_topic_mapping_agent(agent, input_text, list_of_topics):