diff --git a/.github/workflows/azd-ai-template-validation.yml b/.github/workflows/azd-ai-template-validation.yml index 5494447..70c99af 100644 --- a/.github/workflows/azd-ai-template-validation.yml +++ b/.github/workflows/azd-ai-template-validation.yml @@ -1,11 +1,18 @@ name: AZD AI Template validation -# Run when commits are pushed to pre-deploy-alguadam + on: - push: - branches: - - main - - dev - workflow_dispatch: + push: + branches: + - main + - dev + paths: + - 'infra/**' + - 'src/**' + - 'azure.yaml' + - '.github/workflows/azd-ai-template-validation.yml' + workflow_dispatch: + schedule: + - cron: '30 1 * * 4' # Every Thursday 7:00 AM IST / 1:30 AM UTC # Set up permissions for deploying with secretless Azure federated credentials # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication @@ -13,17 +20,30 @@ permissions: id-token: write contents: read +env: + AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} + AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} + RG_TAGS: ${{ vars.RG_TAGS }} + TEMPLATE_USE_DEV_CONTAINER: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }} + TEMPLATE_VALIDATE_AZD: ${{ vars.TEMPLATE_VALIDATE_AZD }} + TEMPLATE_VALIDATE_TESTS: ${{ vars.TEMPLATE_VALIDATE_TESTS }} + AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }} + jobs: validate: runs-on: ubuntu-latest name: Validation steps environment: 'rti-validate' - env: - RG_TAGS: ${{ vars.RG_TAGS }} steps: - name: Checkout uses: actions/checkout@v4 + - name: Set timestamp + shell: bash + run: echo "HHMM=$(date -u +'%H%M')" >> $GITHUB_ENV + - name: Add RG tags into Bicep parameter file shell: bash run: | @@ -45,11 +65,15 @@ jobs: uses: microsoft/template-validation-action@Latest id: validation env: - AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} - AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} - AZURE_ENV_NAME: '${{ vars.AZURE_ENV_NAME }}val' + AZURE_CLIENT_ID: ${{ env.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ env.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ env.AZURE_SUBSCRIPTION_ID }} + AZURE_LOCATION: ${{ env.AZURE_LOCATION }} + AZURE_ENV_NAME: azd-${{ vars.AZURE_ENV_NAME }}-${{ env.HHMM }} + TEMPLATE_USE_DEV_CONTAINER: ${{ env.TEMPLATE_USE_DEV_CONTAINER }} + TEMPLATE_VALIDATE_AZD: ${{ env.TEMPLATE_VALIDATE_AZD }} + TEMPLATE_VALIDATE_TESTS: ${{ env.TEMPLATE_VALIDATE_TESTS }} + AZURE_DEV_COLLECT_TELEMETRY: ${{ env.AZURE_DEV_COLLECT_TELEMETRY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Print result diff --git a/.github/workflows/azure-dev.yml b/.github/workflows/azure-dev.yml index d7f13ed..4393d8d 100644 --- a/.github/workflows/azure-dev.yml +++ b/.github/workflows/azure-dev.yml @@ -1,24 +1,27 @@ name: CI/CD Azure - Real-Time Intelligence Operations -# Trigger the workflow on push to main/master or manual dispatch +# Trigger the workflow on manual dispatch on: workflow_dispatch: push: branches: - main - dev - # - "*" - # paths: - # - "infra/**" - # - "src/**" - # - ".github/workflows/azure-dev.yml" - # pull_request: - # branches: - # - main - # paths: - # - "infra/**" - # - "src/**" - # - ".github/workflows/azure-dev.yml" + paths: + - 'infra/**' + - 'src/**' + - 'azure.yaml' + - 'requirements.txt' + - '.github/workflows/azure-dev.yml' + pull_request: + branches: + - main + paths: + - 'infra/**' + - 'src/**' + - 'azure.yaml' + - 'requirements.txt' + - '.github/workflows/azure-dev.yml' # Set up permissions for deploying with secretless Azure federated credentials permissions: @@ -32,6 +35,10 @@ env: AZURE_LOCATION: 'westus3' PYTHONIOENCODING: utf-8 RG_TAGS: ${{ vars.RG_TAGS }} + TEMPLATE_USE_DEV_CONTAINER: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }} + TEMPLATE_VALIDATE_AZD: ${{ vars.TEMPLATE_VALIDATE_AZD }} + TEMPLATE_VALIDATE_TESTS: ${{ vars.TEMPLATE_VALIDATE_TESTS }} + AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }} jobs: build: @@ -90,9 +97,10 @@ jobs: shell: bash run: | COMMON_PART="rtio" + HHMM=$(date -u +'%H%M') TIMESTAMP=$(date +%s) - UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6) - UNIQUE_ENV_NAME="${COMMON_PART}${UPDATED_TIMESTAMP}" + UPDATED_TIMESTAMP=$(echo "$TIMESTAMP" | tail -c 6) + UNIQUE_ENV_NAME="${COMMON_PART}${HHMM}${UPDATED_TIMESTAMP}" echo "ENV_NAME=${UNIQUE_ENV_NAME}" >> $GITHUB_ENV echo "Generated Environment Name: ${UNIQUE_ENV_NAME}" diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c9f9838..f5bf399 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,18 @@ name: PyLint -on: [push] +on: + push: + paths: + - 'src/**/*.py' + - 'infra/**/*.py' + - 'requirements.txt' + - '.github/workflows/pylint.yml' + pull_request: + paths: + - 'src/**/*.py' + - 'infra/**/*.py' + - 'requirements.txt' + - '.github/workflows/pylint.yml' jobs: build: diff --git a/azure.yaml b/azure.yaml index f508f88..c579718 100644 --- a/azure.yaml +++ b/azure.yaml @@ -5,7 +5,8 @@ metadata: template: real-time-intelligence-operations-solution-accelerator@1.0 requiredVersions: - azd: ">= 1.19.0" + azd: ">= 1.19.0 != 1.23.9" + bicep: '>= 0.33.0' hooks: postprovision: diff --git a/docs/FabricDataAgentGuide.md b/docs/FabricDataAgentGuide.md index 8c3af86..feb0663 100644 --- a/docs/FabricDataAgentGuide.md +++ b/docs/FabricDataAgentGuide.md @@ -6,6 +6,11 @@ After you have deployed your solution, you can add Azure Data Agent to get data - Add the KQL Database created in the Fabric workspace as your data source, - Use the Agent configuration files provided below to set up your Fabric Data Agent. +> [!NOTE] +> The Fabric Data Agent SDK is currently in **Preview**. After deployment, table selections may not persist — you may need to manually select the required tables in the Fabric portal under the **Data** tab and save. Once saved through the UI, the selections persist and the agent works as expected. +> +> ![Data Agent - Tables Selected](../docs/images/deployment/data_agent_tables_selected.png) + ## 📁 Agent Configuration Files This folder contains essential configuration files for setting up your Fabric Data Agent to deliver optimal intelligence based on your data. diff --git a/docs/images/deployment/data_agent_tables_selected.png b/docs/images/deployment/data_agent_tables_selected.png new file mode 100644 index 0000000..1012c8f Binary files /dev/null and b/docs/images/deployment/data_agent_tables_selected.png differ diff --git a/infra/main.bicep b/infra/main.bicep index 086ef30..5ec66ae 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -102,6 +102,10 @@ var allTags = union( tags ) +var eventHubTags = union(allTags, { + SecurityControl: 'Ignore' // Required to override MSFT subscription policy controls that enforce disableLocalAuth; local auth needed for Fabric SAS connection +}) + resource resourceGroupTags 'Microsoft.Resources/tags@2021-04-01' = { name: 'default' properties: { @@ -142,6 +146,7 @@ module eventHubNamespaceModule 'br/public:avm/res/event-hub/namespace:0.13.0' = skuName: 'Standard' skuCapacity: 1 disableLocalAuth: false // NOTE: local auth is currently needed in order to create connection with Fabric via SAS token + tags: eventHubTags eventhubs: [ { name: eventHubName diff --git a/infra/main.parameters.json b/infra/main.parameters.json index faf8053..08b853b 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -10,6 +10,9 @@ }, "existingFabricCapacityName": { "value": "${EXISTING_FABRIC_CAPACITY_NAME}" + }, + "tags": { + "value": "${AZURE_TAGS}" } } } \ No newline at end of file diff --git a/infra/scripts/fabric/fabric_activator_definition.py b/infra/scripts/fabric/fabric_activator_definition.py index ae8f8f3..a4e51e5 100644 --- a/infra/scripts/fabric/fabric_activator_definition.py +++ b/infra/scripts/fabric/fabric_activator_definition.py @@ -32,7 +32,6 @@ import os import re import sys -from typing import Dict, Any, Optional from fabric_api import FabricWorkspaceApiClient, FabricApiError diff --git a/infra/scripts/fabric/fabric_api.py b/infra/scripts/fabric/fabric_api.py index a2c2e1d..577b198 100644 --- a/infra/scripts/fabric/fabric_api.py +++ b/infra/scripts/fabric/fabric_api.py @@ -86,6 +86,22 @@ def _format_duration(self, elapsed_seconds: float) -> str: minutes = int(elapsed_seconds // 60) seconds = int(elapsed_seconds % 60) return f"{minutes}m {seconds}s" + + @staticmethod + def _normalize_eventhub_endpoint(namespace_or_endpoint: str) -> str: + """Normalize Event Hub endpoint to the connector-expected host format. + + Accepts either a bare namespace (e.g. "myns") or a full endpoint + (e.g. "sb://myns.servicebus.windows.net/") and returns + "myns.servicebus.windows.net". + """ + endpoint = (namespace_or_endpoint or "").strip() + if endpoint.startswith("sb://"): + endpoint = endpoint[len("sb://"):] + endpoint = endpoint.rstrip("/") + if ".servicebus.windows.net" not in endpoint: + endpoint = f"{endpoint}.servicebus.windows.net" + return endpoint def start_long_running_operation(self, uri: str, @@ -759,6 +775,8 @@ def create_eventhub_connection(self, name: str, namespace_name: str, event_hub_n """ self._log(f"Creating Event Hub connection: {name}") + normalized_endpoint = self._normalize_eventhub_endpoint(namespace_name) + connection_payload = { "displayName": name, "connectivityType": "ShareableCloud", @@ -770,7 +788,7 @@ def create_eventhub_connection(self, name: str, namespace_name: str, event_hub_n { "name": "endpoint", "dataType": "Text", - "value": namespace_name, + "value": normalized_endpoint, }, { "name": "entityPath", @@ -780,6 +798,7 @@ def create_eventhub_connection(self, name: str, namespace_name: str, event_hub_n ] }, "credentialDetails": { + "singleSignOnType": "None", "credentials": { "credentialType": "Basic", # the endpoint only accepts Basic auth, but takes SAS key with policy name as password and username "username": shared_access_policy_name, #"RootManageSharedAccessKey", @@ -818,11 +837,30 @@ def update_eventhub_connection(self, connection_id: str, name: str, namespace_na try: self._log(f"Updating Event Hub connection: {name} (ID: {connection_id})") + normalized_endpoint = self._normalize_eventhub_endpoint(namespace_name) + connection_payload = { "displayName": name, "connectivityType": "ShareableCloud", "allowConnectionUsageInGateway": False, + "connectionDetails": { + "type": "EventHub", + "creationMethod": "EventHub.Contents", + "parameters": [ + { + "name": "endpoint", + "dataType": "Text", + "value": normalized_endpoint, + }, + { + "name": "entityPath", + "dataType": "Text", + "value": event_hub_name, + } + ] + }, "credentialDetails": { + "singleSignOnType": "None", "credentials": { "credentialType": "Basic", # the endpoint only accepts Basic auth, but takes SAS key with policy name as password and username "username": shared_access_policy_name, diff --git a/infra/scripts/fabric/fabric_common_utils.py b/infra/scripts/fabric/fabric_common_utils.py index ab61bb1..ef7bcd7 100644 --- a/infra/scripts/fabric/fabric_common_utils.py +++ b/infra/scripts/fabric/fabric_common_utils.py @@ -13,7 +13,6 @@ import os import sys -import argparse from datetime import datetime def get_required_env_var(var_name: str) -> str: diff --git a/infra/scripts/fabric/fabric_database.py b/infra/scripts/fabric/fabric_database.py index 63b79aa..974ebe7 100644 --- a/infra/scripts/fabric/fabric_database.py +++ b/infra/scripts/fabric/fabric_database.py @@ -14,8 +14,6 @@ - Access to the specified Fabric cluster and database """ -import argparse -import os import sys from pathlib import Path @@ -25,7 +23,7 @@ sys.path.insert(0, str(scripts_dir)) from azure.identity import AzureCliCredential -from azure.kusto.data import KustoClient, KustoConnectionStringBuilder, ClientRequestProperties +from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from azure.kusto.data.exceptions import KustoServiceError def create_kusto_client(cluster_uri) -> KustoClient: diff --git a/infra/scripts/fabric/fabric_eventhouse.py b/infra/scripts/fabric/fabric_eventhouse.py index b5e12d8..05919a6 100644 --- a/infra/scripts/fabric/fabric_eventhouse.py +++ b/infra/scripts/fabric/fabric_eventhouse.py @@ -18,7 +18,7 @@ import argparse import sys import time -from typing import Optional, Dict, Any +from typing import Optional from fabric_api import FabricWorkspaceApiClient, FabricApiError diff --git a/infra/scripts/fabric/fabric_eventhub.py b/infra/scripts/fabric/fabric_eventhub.py index cc657e0..d812531 100644 --- a/infra/scripts/fabric/fabric_eventhub.py +++ b/infra/scripts/fabric/fabric_eventhub.py @@ -27,7 +27,6 @@ """ import argparse -import sys from azure.identity import AzureCliCredential from azure.mgmt.eventhub import EventHubManagementClient from fabric_api import FabricApiClient, FabricApiError diff --git a/infra/scripts/fabric/fabric_eventstream_definition.py b/infra/scripts/fabric/fabric_eventstream_definition.py index e8e8a83..57d47cb 100644 --- a/infra/scripts/fabric/fabric_eventstream_definition.py +++ b/infra/scripts/fabric/fabric_eventstream_definition.py @@ -24,7 +24,6 @@ import json import os import sys -from typing import Dict, Any, Optional from fabric_api import FabricWorkspaceApiClient, FabricApiError from fabric_auth import authenticate_workspace diff --git a/infra/scripts/fabric/fabric_workspace.py b/infra/scripts/fabric/fabric_workspace.py index f6281b8..58719f2 100644 --- a/infra/scripts/fabric/fabric_workspace.py +++ b/infra/scripts/fabric/fabric_workspace.py @@ -16,7 +16,6 @@ """ import argparse -import sys from fabric_api import FabricApiClient, FabricWorkspaceApiClient, FabricApiError def setup_workspace(fabric_client: FabricApiClient, capacity_name: str, workspace_name: str) -> str: diff --git a/infra/scripts/fabric/graph_api.py b/infra/scripts/fabric/graph_api.py index a524270..09f3e07 100644 --- a/infra/scripts/fabric/graph_api.py +++ b/infra/scripts/fabric/graph_api.py @@ -22,7 +22,7 @@ import time import uuid from datetime import datetime, timedelta -from typing import Dict, List, Optional, Union, Any, Tuple +from typing import Dict, Optional, Union, Any, Tuple from azure.identity import AzureCliCredential