Merge pull request #65 from microsoft/dev

Avijit-Microsoft · web-flow · commit b345a3513f58 · 2026-04-20T11:01:20.000+05:30
chore: dev to main merge
diff --git a/.github/workflows/azd-ai-template-validation.yml b/.github/workflows/azd-ai-template-validation.yml
@@ -1,29 +1,49 @@
 name: AZD AI Template validation
-# Run when commits are pushed to pre-deploy-alguadam
+
 on:
-    push:
-      branches:
-        - main
-        - dev
-    workflow_dispatch:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - 'infra/**'
+      - 'src/**'
+      - 'azure.yaml'
+      - '.github/workflows/azd-ai-template-validation.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '30 1 * * 4' # Every Thursday 7:00 AM IST / 1:30 AM UTC
 
 # Set up permissions for deploying with secretless Azure federated credentials
 # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
 permissions:
   id-token: write
   contents: read
 
+env:
+  AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
+  AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
+  AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+  AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+  RG_TAGS: ${{ vars.RG_TAGS }}
+  TEMPLATE_USE_DEV_CONTAINER: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }}
+  TEMPLATE_VALIDATE_AZD: ${{ vars.TEMPLATE_VALIDATE_AZD }}
+  TEMPLATE_VALIDATE_TESTS: ${{ vars.TEMPLATE_VALIDATE_TESTS }}
+  AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
+
 jobs:
   validate:
     runs-on: ubuntu-latest
     name: Validation steps
     environment: 'rti-validate'
-    env:
-      RG_TAGS: ${{ vars.RG_TAGS }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Set timestamp
+        shell: bash
+        run: echo "HHMM=$(date -u +'%H%M')" >> $GITHUB_ENV
+
       - name: Add RG tags into Bicep parameter file
         shell: bash
         run: |
@@ -45,11 +65,15 @@ jobs:
         uses: microsoft/template-validation-action@Latest
         id: validation
         env:
-          AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
-          AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
-          AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
-          AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
-          AZURE_ENV_NAME: '${{ vars.AZURE_ENV_NAME }}val'
+          AZURE_CLIENT_ID: ${{ env.AZURE_CLIENT_ID }}
+          AZURE_TENANT_ID: ${{ env.AZURE_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ env.AZURE_SUBSCRIPTION_ID }}
+          AZURE_LOCATION: ${{ env.AZURE_LOCATION }}
+          AZURE_ENV_NAME: azd-${{ vars.AZURE_ENV_NAME }}-${{ env.HHMM }}
+          TEMPLATE_USE_DEV_CONTAINER: ${{ env.TEMPLATE_USE_DEV_CONTAINER }}
+          TEMPLATE_VALIDATE_AZD: ${{ env.TEMPLATE_VALIDATE_AZD }}
+          TEMPLATE_VALIDATE_TESTS: ${{ env.TEMPLATE_VALIDATE_TESTS }}
+          AZURE_DEV_COLLECT_TELEMETRY: ${{ env.AZURE_DEV_COLLECT_TELEMETRY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Print result
diff --git a/.github/workflows/azure-dev.yml b/.github/workflows/azure-dev.yml
@@ -1,24 +1,27 @@
 name: CI/CD Azure - Real-Time Intelligence Operations
 
-# Trigger the workflow on push to main/master or manual dispatch
+# Trigger the workflow on manual dispatch
 on:
   workflow_dispatch:
   push:
     branches:
       - main
       - dev
-  #     - "*"
-  #   paths:
-  #     - "infra/**"
-  #     - "src/**"
-  #     - ".github/workflows/azure-dev.yml"
-  # pull_request:
-  #   branches:
-  #     - main
-  #   paths:
-  #     - "infra/**"
-  #     - "src/**"
-  #     - ".github/workflows/azure-dev.yml"
+    paths:
+      - 'infra/**'
+      - 'src/**'
+      - 'azure.yaml'
+      - 'requirements.txt'
+      - '.github/workflows/azure-dev.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'infra/**'
+      - 'src/**'
+      - 'azure.yaml'
+      - 'requirements.txt'
+      - '.github/workflows/azure-dev.yml'
 
 # Set up permissions for deploying with secretless Azure federated credentials
 permissions:
@@ -32,6 +35,10 @@ env:
   AZURE_LOCATION: 'westus3'
   PYTHONIOENCODING: utf-8
   RG_TAGS: ${{ vars.RG_TAGS }}
+  TEMPLATE_USE_DEV_CONTAINER: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }}
+  TEMPLATE_VALIDATE_AZD: ${{ vars.TEMPLATE_VALIDATE_AZD }}
+  TEMPLATE_VALIDATE_TESTS: ${{ vars.TEMPLATE_VALIDATE_TESTS }}
+  AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
   
 jobs:
   build:
@@ -90,9 +97,10 @@ jobs:
         shell: bash
         run: |
           COMMON_PART="rtio"
+          HHMM=$(date -u +'%H%M')
           TIMESTAMP=$(date +%s)
-          UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6)
-          UNIQUE_ENV_NAME="${COMMON_PART}${UPDATED_TIMESTAMP}"
+          UPDATED_TIMESTAMP=$(echo "$TIMESTAMP" | tail -c 6)
+          UNIQUE_ENV_NAME="${COMMON_PART}${HHMM}${UPDATED_TIMESTAMP}"
           echo "ENV_NAME=${UNIQUE_ENV_NAME}" >> $GITHUB_ENV
           echo "Generated Environment Name: ${UNIQUE_ENV_NAME}"
           
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -1,6 +1,18 @@
 name: PyLint
 
-on: [push]
+on:
+  push:
+    paths:
+      - 'src/**/*.py'
+      - 'infra/**/*.py'
+      - 'requirements.txt'
+      - '.github/workflows/pylint.yml'
+  pull_request:
+    paths:
+      - 'src/**/*.py'
+      - 'infra/**/*.py'
+      - 'requirements.txt'
+      - '.github/workflows/pylint.yml'
 
 jobs:
   build:
diff --git a/azure.yaml b/azure.yaml
@@ -5,7 +5,8 @@ metadata:
   template: real-time-intelligence-operations-solution-accelerator@1.0
 
 requiredVersions:
-  azd: ">= 1.19.0"
+  azd: ">= 1.19.0 != 1.23.9"
+  bicep: '>= 0.33.0'
 
 hooks:
   postprovision:
diff --git a/docs/FabricDataAgentGuide.md b/docs/FabricDataAgentGuide.md
@@ -6,6 +6,11 @@ After you have deployed your solution, you can add Azure Data Agent to get data
 - Add the KQL Database created in the Fabric workspace as your data source,
 - Use the Agent configuration files provided below to set up your Fabric Data Agent. 
 
+> [!NOTE]
+> The Fabric Data Agent SDK is currently in **Preview**. After deployment, table selections may not persist — you may need to manually select the required tables in the Fabric portal under the **Data** tab and save. Once saved through the UI, the selections persist and the agent works as expected.
+>
+> ![Data Agent - Tables Selected](../docs/images/deployment/data_agent_tables_selected.png)
+
 ## 📁 Agent Configuration Files
 
 This folder contains essential configuration files for setting up your Fabric Data Agent to deliver optimal intelligence based on your data.
diff --git a/docs/images/deployment/data_agent_tables_selected.png b/docs/images/deployment/data_agent_tables_selected.png
diff --git a/infra/main.bicep b/infra/main.bicep
@@ -102,6 +102,10 @@ var allTags = union(
   tags
 )
 
+var eventHubTags = union(allTags, {
+  SecurityControl: 'Ignore' // Required to override MSFT subscription policy controls that enforce disableLocalAuth; local auth needed for Fabric SAS connection
+})
+
 resource resourceGroupTags 'Microsoft.Resources/tags@2021-04-01' = {
   name: 'default'
   properties: {
@@ -142,6 +146,7 @@ module eventHubNamespaceModule 'br/public:avm/res/event-hub/namespace:0.13.0' =
     skuName: 'Standard'
     skuCapacity: 1
     disableLocalAuth: false // NOTE: local auth is currently needed in order to create connection with Fabric via SAS token
+    tags: eventHubTags
     eventhubs: [
       {
         name: eventHubName
diff --git a/infra/main.parameters.json b/infra/main.parameters.json
@@ -10,6 +10,9 @@
     },
     "existingFabricCapacityName": {
       "value": "${EXISTING_FABRIC_CAPACITY_NAME}"
+    },
+    "tags": {
+      "value": "${AZURE_TAGS}"
     }
   }
 }
diff --git a/infra/scripts/fabric/fabric_activator_definition.py b/infra/scripts/fabric/fabric_activator_definition.py
@@ -32,7 +32,6 @@
 import os
 import re
 import sys
-from typing import Dict, Any, Optional
 
 from fabric_api import FabricWorkspaceApiClient, FabricApiError
 
diff --git a/infra/scripts/fabric/fabric_api.py b/infra/scripts/fabric/fabric_api.py
@@ -86,6 +86,22 @@ def _format_duration(self, elapsed_seconds: float) -> str:
         minutes = int(elapsed_seconds // 60)
         seconds = int(elapsed_seconds % 60)
         return f"{minutes}m {seconds}s"
+
+    @staticmethod
+    def _normalize_eventhub_endpoint(namespace_or_endpoint: str) -> str:
+        """Normalize Event Hub endpoint to the connector-expected host format.
+
+        Accepts either a bare namespace (e.g. "myns") or a full endpoint
+        (e.g. "sb://myns.servicebus.windows.net/") and returns
+        "myns.servicebus.windows.net".
+        """
+        endpoint = (namespace_or_endpoint or "").strip()
+        if endpoint.startswith("sb://"):
+            endpoint = endpoint[len("sb://"):]
+        endpoint = endpoint.rstrip("/")
+        if ".servicebus.windows.net" not in endpoint:
+            endpoint = f"{endpoint}.servicebus.windows.net"
+        return endpoint
     
     def start_long_running_operation(self,
                                    uri: str,
@@ -759,6 +775,8 @@ def create_eventhub_connection(self, name: str, namespace_name: str, event_hub_n
         """
         self._log(f"Creating Event Hub connection: {name}")
         
+        normalized_endpoint = self._normalize_eventhub_endpoint(namespace_name)
+
         connection_payload = {
             "displayName": name,
             "connectivityType": "ShareableCloud",
@@ -770,7 +788,7 @@ def create_eventhub_connection(self, name: str, namespace_name: str, event_hub_n
                     {
                         "name": "endpoint",
                         "dataType": "Text",
-                        "value": namespace_name,
+                        "value": normalized_endpoint,
                     },
                     {
                         "name": "entityPath",
@@ -780,6 +798,7 @@ def create_eventhub_connection(self, name: str, namespace_name: str, event_hub_n
                 ]
             },
             "credentialDetails": {
+                "singleSignOnType": "None",
                 "credentials": {
                     "credentialType": "Basic", # the endpoint only accepts Basic auth, but takes SAS key with policy name as password and username
                     "username": shared_access_policy_name, #"RootManageSharedAccessKey",
@@ -818,11 +837,30 @@ def update_eventhub_connection(self, connection_id: str, name: str, namespace_na
         try:
             self._log(f"Updating Event Hub connection: {name} (ID: {connection_id})")
             
+            normalized_endpoint = self._normalize_eventhub_endpoint(namespace_name)
+
             connection_payload = {
                 "displayName": name,
                 "connectivityType": "ShareableCloud",
                 "allowConnectionUsageInGateway": False,
+                "connectionDetails": {
+                    "type": "EventHub",
+                    "creationMethod": "EventHub.Contents",
+                    "parameters": [
+                        {
+                            "name": "endpoint",
+                            "dataType": "Text",
+                            "value": normalized_endpoint,
+                        },
+                        {
+                            "name": "entityPath",
+                            "dataType": "Text",
+                            "value": event_hub_name,
+                        }
+                    ]
+                },
                 "credentialDetails": {
+                    "singleSignOnType": "None",
                     "credentials": {
                         "credentialType": "Basic", # the endpoint only accepts Basic auth, but takes SAS key with policy name as password and username
                         "username": shared_access_policy_name,
diff --git a/infra/scripts/fabric/fabric_common_utils.py b/infra/scripts/fabric/fabric_common_utils.py
@@ -13,7 +13,6 @@
 
 import os
 import sys
-import argparse
 from datetime import datetime
 
 def get_required_env_var(var_name: str) -> str:
diff --git a/infra/scripts/fabric/fabric_database.py b/infra/scripts/fabric/fabric_database.py
@@ -14,8 +14,6 @@
     - Access to the specified Fabric cluster and database
 """
 
-import argparse
-import os
 import sys
 from pathlib import Path
 
@@ -25,7 +23,7 @@
 sys.path.insert(0, str(scripts_dir))
 
 from azure.identity import AzureCliCredential
-from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties
+from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
 from azure.kusto.data.exceptions import KustoServiceError
 
 def create_kusto_client(cluster_uri) -> KustoClient:
diff --git a/infra/scripts/fabric/fabric_eventhouse.py b/infra/scripts/fabric/fabric_eventhouse.py
@@ -18,7 +18,7 @@
 import argparse
 import sys
 import time
-from typing import Optional, Dict, Any
+from typing import Optional
 from fabric_api import FabricWorkspaceApiClient, FabricApiError
 
 
diff --git a/infra/scripts/fabric/fabric_eventhub.py b/infra/scripts/fabric/fabric_eventhub.py
@@ -27,7 +27,6 @@
 """
 
 import argparse
-import sys
 from azure.identity import AzureCliCredential
 from azure.mgmt.eventhub import EventHubManagementClient
 from fabric_api import FabricApiClient, FabricApiError
diff --git a/infra/scripts/fabric/fabric_eventstream_definition.py b/infra/scripts/fabric/fabric_eventstream_definition.py
@@ -24,7 +24,6 @@
 import json
 import os
 import sys
-from typing import Dict, Any, Optional
 from fabric_api import FabricWorkspaceApiClient, FabricApiError
 from fabric_auth import authenticate_workspace
 
diff --git a/infra/scripts/fabric/fabric_workspace.py b/infra/scripts/fabric/fabric_workspace.py
@@ -16,7 +16,6 @@
 """
 
 import argparse
-import sys
 from fabric_api import FabricApiClient, FabricWorkspaceApiClient, FabricApiError
 
 def setup_workspace(fabric_client: FabricApiClient, capacity_name: str, workspace_name: str) -> str:
diff --git a/infra/scripts/fabric/graph_api.py b/infra/scripts/fabric/graph_api.py
@@ -22,7 +22,7 @@
 import time
 import uuid
 from datetime import datetime, timedelta
-from typing import Dict, List, Optional, Union, Any, Tuple
+from typing import Dict, Optional, Union, Any, Tuple
 from azure.identity import AzureCliCredential
 
 

Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,9 @@`
`10`	`10`	`},`
`11`	`11`	`"existingFabricCapacityName": {`
`12`	`12`	`"value": "${EXISTING_FABRIC_CAPACITY_NAME}"`
	`13`	`+ },`
	`14`	`+ "tags": {`
	`15`	`+ "value": "${AZURE_TAGS}"`
`13`	`16`	`}`
`14`	`17`	`}`
`15`	`18`	`}`