From c9c00d7720cfaafd079157a1ebafb55d0cc05788 Mon Sep 17 00:00:00 2001 From: Josiel Souza Date: Tue, 12 May 2026 09:19:54 +0100 Subject: [PATCH] DTOSS-12832: Enable App Insights telemetry and Arc MI relay authentication - Add app-insights module (workspace-linked to Arc LAW) to arc-infra - Surface connection string as a Terraform output - Fetch connection string at deploy time via az CLI and inject into .env - Derive CLOUD_API_ENDPOINT from per-environment CLOUD_API_HOSTNAME - Grant each Arc machine's system-assigned MI the Azure Relay Listener role on its own HC, removing the need for SAS keys - Add azureRelayListener to rbacAdmin condition allowlist in core.bicep - Remove AZURE_RELAY_KEY_NAME, AZURE_RELAY_SHARED_ACCESS_KEY, CLOUD_API_TOKEN from .env (superseded by managed identity) --- infrastructure/environments/dev/variables.sh | 1 + .../environments/preprod/variables.sh | 1 + infrastructure/environments/prod/variables.sh | 1 + .../environments/review/variables.sh | 1 + .../modules/arc-infra/azure_monitor.tf | 11 ++++ infrastructure/modules/arc-infra/outputs.tf | 6 ++ infrastructure/modules/arc-infra/relay.tf | 24 +++++++- infrastructure/terraform/outputs.tf | 6 ++ .../terraform/resource_group_init/core.bicep | 5 +- scripts/bash/deploy_arc_ring.sh | 61 +++++-------------- scripts/bash/deploy_stage.sh | 5 +- 11 files changed, 67 insertions(+), 55 deletions(-) diff --git a/infrastructure/environments/dev/variables.sh b/infrastructure/environments/dev/variables.sh index 4219024c..085d3935 100644 --- a/infrastructure/environments/dev/variables.sh +++ b/infrastructure/environments/dev/variables.sh @@ -6,3 +6,4 @@ HUB=dev TERRAFORM_MODULES_REF=main ENABLE_SOFT_DELETE=false ADO_MANAGEMENT_POOL=private-pool-dev-uks +CLOUD_API_HOSTNAME=dev.manage-breast-screening.non-live.screening.nhs.uk diff --git a/infrastructure/environments/preprod/variables.sh b/infrastructure/environments/preprod/variables.sh index e0025410..f53bbbad 100644 --- a/infrastructure/environments/preprod/variables.sh +++ b/infrastructure/environments/preprod/variables.sh @@ -6,3 +6,4 @@ HUB=prod TERRAFORM_MODULES_REF=main ENABLE_SOFT_DELETE=true ADO_MANAGEMENT_POOL=private-pool-prod-uks +CLOUD_API_HOSTNAME=preprod.manage-breast-screening.nhs.uk diff --git a/infrastructure/environments/prod/variables.sh b/infrastructure/environments/prod/variables.sh index 14d6883b..1dc2da77 100644 --- a/infrastructure/environments/prod/variables.sh +++ b/infrastructure/environments/prod/variables.sh @@ -6,4 +6,5 @@ HUB=prod TERRAFORM_MODULES_REF=main ENABLE_SOFT_DELETE=true ADO_MANAGEMENT_POOL=private-pool-prod-uks +CLOUD_API_HOSTNAME=manage-breast-screening.nhs.uk # To onboard more rings: set GATEWAY_RINGS="ring0 ring1 ring2 ..." in this file diff --git a/infrastructure/environments/review/variables.sh b/infrastructure/environments/review/variables.sh index e9c6e841..9f9779a5 100644 --- a/infrastructure/environments/review/variables.sh +++ b/infrastructure/environments/review/variables.sh @@ -6,3 +6,4 @@ HUB=dev TERRAFORM_MODULES_REF=main ENABLE_SOFT_DELETE=false ADO_MANAGEMENT_POOL=private-pool-dev-uks +CLOUD_API_HOSTNAME=review.manage-breast-screening.non-live.screening.nhs.uk diff --git a/infrastructure/modules/arc-infra/azure_monitor.tf b/infrastructure/modules/arc-infra/azure_monitor.tf index d787826c..d6bb1412 100644 --- a/infrastructure/modules/arc-infra/azure_monitor.tf +++ b/infrastructure/modules/arc-infra/azure_monitor.tf @@ -217,3 +217,14 @@ resource "azurerm_resource_group_policy_remediation" "wac" { policy_assignment_id = azurerm_resource_group_policy_assignment.wac[0].id resource_discovery_mode = "ReEvaluateCompliance" } + +module "app_insights" { + count = var.enable_arc_servers ? 1 : 0 + source = "../dtos-devops-templates/infrastructure/modules/app-insights" + + name = "ai-${var.app_short_name}-${var.env_config}-arc-uks" + location = var.region + resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name + appinsights_type = "other" + log_analytics_workspace_id = module.log_analytics_workspace[0].id +} diff --git a/infrastructure/modules/arc-infra/outputs.tf b/infrastructure/modules/arc-infra/outputs.tf index 4173dbe8..5030ce4f 100644 --- a/infrastructure/modules/arc-infra/outputs.tf +++ b/infrastructure/modules/arc-infra/outputs.tf @@ -28,6 +28,12 @@ output "relay_namespace_hostname" { value = var.enable_arc_servers ? "${local.relay_namespace_name}.servicebus.windows.net" : null } +output "app_insights_connection_string" { + description = "Application Insights connection string for the gateway services" + sensitive = true + value = var.enable_arc_servers ? module.app_insights[0].connection_string : null +} + output "relay_listen_sas_keys" { description = "Per-machine relay listen SAS primary keys, keyed by Arc resource name. Used by the deploy pipeline to write .env files." sensitive = true diff --git a/infrastructure/modules/arc-infra/relay.tf b/infrastructure/modules/arc-infra/relay.tf index 5c45d042..db8a945f 100644 --- a/infrastructure/modules/arc-infra/relay.tf +++ b/infrastructure/modules/arc-infra/relay.tf @@ -45,8 +45,8 @@ resource "azurerm_relay_hybrid_connection" "per_machine" { requires_client_authorization = true } -# Listen-only SAS rule per HC — distributed to each gateway site via the deploy pipeline. -# The cloud app uses a namespace-level Send SAS and does not need per-site keys. +# Listen-only SAS rule per HC — retained for local development / break-glass access. +# Production relay authentication uses Managed Identity (see relay_listener_role below). resource "azurerm_relay_hybrid_connection_authorization_rule" "per_machine_listen" { for_each = local.arc_machines @@ -59,3 +59,23 @@ resource "azurerm_relay_hybrid_connection_authorization_rule" "per_machine_liste send = false manage = false } + +# Look up each discovered Arc machine to obtain its system-assigned managed identity. +# Static machines (registered in the same apply) are excluded — they are not yet +# visible to the data source and will be picked up on the next apply after onboarding. +data "azurerm_arc_machine" "machines" { + for_each = local.arc_machines_discovered + name = each.key + resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name +} + +# Grant each machine's MI the Azure Relay Listener role on its own HC so the relay +# listener service can authenticate without a SAS key. +module "relay_listener_role" { + for_each = local.arc_machines_discovered + source = "../dtos-devops-templates/infrastructure/modules/rbac-assignment" + + scope = azurerm_relay_hybrid_connection.per_machine[each.key].id + role_definition_name = "Azure Relay Listener" + principal_id = data.azurerm_arc_machine.machines[each.key].identity[0].principal_id +} diff --git a/infrastructure/terraform/outputs.tf b/infrastructure/terraform/outputs.tf index cb474a26..b1142a91 100644 --- a/infrastructure/terraform/outputs.tf +++ b/infrastructure/terraform/outputs.tf @@ -1,3 +1,9 @@ +output "app_insights_connection_string" { + description = "Application Insights connection string for the gateway services" + sensitive = true + value = module.arc_infra.app_insights_connection_string +} + output "relay_namespace_hostname" { description = "Relay namespace FQDN for AZURE_RELAY_NAMESPACE in the gateway .env" value = module.arc_infra.relay_namespace_hostname diff --git a/infrastructure/terraform/resource_group_init/core.bicep b/infrastructure/terraform/resource_group_init/core.bicep index 79a8c165..09d31e89 100644 --- a/infrastructure/terraform/resource_group_init/core.bicep +++ b/infrastructure/terraform/resource_group_init/core.bicep @@ -36,6 +36,7 @@ var roleID = { resourcePolicyContributor: '36243c78-bf99-498c-9df9-86d9f8d28608' virtualMachineAdministratorLogin: '1c0163c0-47e6-4577-8991-ea5c82e286e4' windowsAdminCenterAdministratorLogin: 'a6333a3e-0164-44c3-b281-7a577aff287f' + azureRelayListener: '26e0b698-aa6d-4085-9386-aadae190014d' } // Define role assignments for managed identity @@ -61,7 +62,7 @@ var miRoleAssignments = [ description: 'RBAC Administrator. Restricted to only assign/remove: Storage Blob Data Contributor, Storage Queue Data Contributor, Azure Connected Machine Onboarding, Azure Connected Machine Resource Administrator, Log Analytics Contributor, Virtual Machine Administrator Login, and Windows Admin Center Administrator Login.' // Delegated RBAC: This condition restricts the RBAC Administrator to only manage specific roles. // This is a security best practice that prevents the identity from granting itself or others sensitive roles like 'Owner' or 'User Access Administrator'. - condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}}))' + condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}}))' conditionVersion: '2.0' } ] @@ -84,7 +85,7 @@ var groupRoleAssignments = [ description: 'RBAC Administrator. Restricted to only assign/remove: Storage Blob Data Contributor, Storage Queue Data Contributor, Azure Connected Machine Onboarding, Azure Connected Machine Resource Administrator, Log Analytics Contributor, Virtual Machine Administrator Login, and Windows Admin Center Administrator Login.' // Delegated RBAC: This condition restricts the RBAC Administrator to only manage specific roles. // This is a security best practice that prevents the identity from granting itself or others sensitive roles like 'Owner' or 'User Access Administrator'. - condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}}))' + condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}}))' conditionVersion: '2.0' } ] diff --git a/scripts/bash/deploy_arc_ring.sh b/scripts/bash/deploy_arc_ring.sh index dcb213d3..3455e527 100755 --- a/scripts/bash/deploy_arc_ring.sh +++ b/scripts/bash/deploy_arc_ring.sh @@ -2,31 +2,18 @@ # Deploy the gateway app to all Arc machines matching a ring within an environment. # Called by deploy_stage.sh. # -# Usage: deploy_arc_ring.sh +# Usage: deploy_arc_ring.sh set -euo pipefail ENVIRONMENT=$1 RING=$2 RELEASE_TAG=$3 -KV_NAME=$4 APP_SHORT_NAME="mbsgw" ARC_RG="rg-${APP_SHORT_NAME}-${ENVIRONMENT}-uks-arc-enabled-servers" -# Relay namespace is owned by dtos-manage-breast-screening; derive from environment name. -RELAY_NAMESPACE_NAME="relay-manbrs-${ENVIRONMENT}" -RELAY_RG="rg-manbrs-${ENVIRONMENT}-uks" -RELAY_NAMESPACE_HOSTNAME="${RELAY_NAMESPACE_NAME}.servicebus.windows.net" - -# Ensure the relay extension is installed -if ! az relay --help &>/dev/null; then - echo "Installing Azure CLI 'relay' extension..." - az extension add --name relay || { - echo "ERROR: Failed to install 'relay' extension. Please run 'az extension add --name relay' manually." - exit 1 - } -fi +RELAY_NAMESPACE_HOSTNAME="relay-manbrs-${ENVIRONMENT}.servicebus.windows.net" # Use forward slashes — Python handles these fine on Windows and avoids .env escaping issues BASE_PATH="C:/Program Files/NHS/ManageBreastScreeningGateway" @@ -34,6 +21,17 @@ PYTHON_VERSION=$(awk '/^python / {print $2}' .tool-versions) echo "--- Ring: ${RING} | Environment: ${ENVIRONMENT} | Release: ${RELEASE_TAG} ---" +# ── Per-environment config ───────────────────────────────────────────────────── +source "infrastructure/environments/${ENVIRONMENT}/variables.sh" +CLOUD_API_ENDPOINT="https://${CLOUD_API_HOSTNAME}/api/v1/dicom" + +APPLICATIONINSIGHTS_CONNECTION_STRING=$(az monitor app-insights component show \ + --app "ai-${APP_SHORT_NAME}-${ENVIRONMENT}-arc-uks" \ + --resource-group "$ARC_RG" \ + --query connectionString -o tsv 2>/dev/null || echo "") +[[ -z "$APPLICATIONINSIGHTS_CONNECTION_STRING" ]] && \ + echo "##vso[task.logissue type=warning]Application Insights resource not found — telemetry will be disabled" + # ── Discover machines ────────────────────────────────────────────────────────── MACHINES_JSON=$(az connectedmachine list \ --resource-group "$ARC_RG" \ @@ -61,44 +59,13 @@ while IFS= read -r MACHINE_JSON; do LOCATION=$(echo "$MACHINE_JSON" | jq -r '.location') echo "Preparing deploy for $MACHINE ($LOCATION)..." - # Fetch relay SAS key directly — Contributor includes listKeys on relay HCs, - # and this avoids any dependency on Terraform state having the resource imported. - echo "Fetching SAS key for hc-${MACHINE} in $RELAY_NAMESPACE_NAME..." - SAS_KEY=$(az relay hyco authorization-rule keys list \ - --resource-group "$RELAY_RG" \ - --namespace-name "$RELAY_NAMESPACE_NAME" \ - --hybrid-connection-name "hc-${MACHINE}" \ - --name listen \ - --query primaryKey -o tsv 2>/tmp/relay_key_err_${MACHINE}) || { - ERR=$(cat /tmp/relay_key_err_${MACHINE}) - echo "##vso[task.logissue type=warning]Failed to fetch relay SAS key for hc-${MACHINE}: $ERR" - SAS_KEY="" - } - - [[ -z "$SAS_KEY" ]] && \ - echo "##vso[task.logissue type=warning]No relay SAS key found for hc-${MACHINE} — relay listener will not connect" - - # Cloud API secrets are optional — warn if absent, services still start - CLOUD_API_ENDPOINT=$(az keyvault secret show --vault-name "$KV_NAME" \ - --name "cloud-api-endpoint" --query value -o tsv 2>/dev/null || echo "") - CLOUD_API_TOKEN=$(az keyvault secret show --vault-name "$KV_NAME" \ - --name "cloud-api-token-${MACHINE}" --query value -o tsv 2>/dev/null || echo "") - - [[ -z "$CLOUD_API_ENDPOINT" ]] && \ - echo "##vso[task.logissue type=warning]cloud-api-endpoint not in $KV_NAME — Upload service will not reach cloud API for $MACHINE" - [[ -z "$CLOUD_API_TOKEN" ]] && \ - echo "##vso[task.logissue type=warning]cloud-api-token-${MACHINE} not in $KV_NAME — Upload service will not authenticate for $MACHINE" - # Build .env, then base64-encode to pass newlines as a run command parameter. # NOTE: Arc Run Command drops protectedParameters for inline source.script, # so EnvContentB64 travels as a regular parameter (base64-encoded, not plain text). - # TODO: migrate to Key Vault + Arc MSI for production environments. ENV_CONTENT="AZURE_RELAY_NAMESPACE=${RELAY_NAMESPACE_HOSTNAME} AZURE_RELAY_HYBRID_CONNECTION=hc-${MACHINE} -AZURE_RELAY_KEY_NAME=listen -AZURE_RELAY_SHARED_ACCESS_KEY=${SAS_KEY} CLOUD_API_ENDPOINT=${CLOUD_API_ENDPOINT} -CLOUD_API_TOKEN=${CLOUD_API_TOKEN} +APPLICATIONINSIGHTS_CONNECTION_STRING=${APPLICATIONINSIGHTS_CONNECTION_STRING} MWL_AET=SCREENING_MWL MWL_PORT=4243 MWL_DB_PATH=${BASE_PATH}/data/worklist.db diff --git a/scripts/bash/deploy_stage.sh b/scripts/bash/deploy_stage.sh index 807f4ced..ece2bf0c 100755 --- a/scripts/bash/deploy_stage.sh +++ b/scripts/bash/deploy_stage.sh @@ -12,9 +12,6 @@ ENVIRONMENT=$1 RINGS=$2 RELEASE_TAG=$3 -APP_SHORT_NAME="mbsgw" -KV_NAME="kv-${APP_SHORT_NAME}-${ENVIRONMENT}-inf" - echo "========================================" echo "Environment : ${ENVIRONMENT}" echo "Rings : ${RINGS}" @@ -22,5 +19,5 @@ echo "Release : ${RELEASE_TAG}" echo "========================================" for RING in $RINGS; do - scripts/bash/deploy_arc_ring.sh "$ENVIRONMENT" "$RING" "$RELEASE_TAG" "$KV_NAME" + scripts/bash/deploy_arc_ring.sh "$ENVIRONMENT" "$RING" "$RELEASE_TAG" done