Skip to content

Commit c9c00d7

Browse files
DTOSS-12832: Enable App Insights telemetry and Arc MI relay authentication
- Add app-insights module (workspace-linked to Arc LAW) to arc-infra - Surface connection string as a Terraform output - Fetch connection string at deploy time via az CLI and inject into .env - Derive CLOUD_API_ENDPOINT from per-environment CLOUD_API_HOSTNAME - Grant each Arc machine's system-assigned MI the Azure Relay Listener role on its own HC, removing the need for SAS keys - Add azureRelayListener to rbacAdmin condition allowlist in core.bicep - Remove AZURE_RELAY_KEY_NAME, AZURE_RELAY_SHARED_ACCESS_KEY, CLOUD_API_TOKEN from .env (superseded by managed identity)
1 parent 8836ca8 commit c9c00d7

11 files changed

Lines changed: 67 additions & 55 deletions

File tree

infrastructure/environments/dev/variables.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ HUB=dev
66
TERRAFORM_MODULES_REF=main
77
ENABLE_SOFT_DELETE=false
88
ADO_MANAGEMENT_POOL=private-pool-dev-uks
9+
CLOUD_API_HOSTNAME=dev.manage-breast-screening.non-live.screening.nhs.uk

infrastructure/environments/preprod/variables.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ HUB=prod
66
TERRAFORM_MODULES_REF=main
77
ENABLE_SOFT_DELETE=true
88
ADO_MANAGEMENT_POOL=private-pool-prod-uks
9+
CLOUD_API_HOSTNAME=preprod.manage-breast-screening.nhs.uk

infrastructure/environments/prod/variables.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ HUB=prod
66
TERRAFORM_MODULES_REF=main
77
ENABLE_SOFT_DELETE=true
88
ADO_MANAGEMENT_POOL=private-pool-prod-uks
9+
CLOUD_API_HOSTNAME=manage-breast-screening.nhs.uk
910
# To onboard more rings: set GATEWAY_RINGS="ring0 ring1 ring2 ..." in this file

infrastructure/environments/review/variables.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ HUB=dev
66
TERRAFORM_MODULES_REF=main
77
ENABLE_SOFT_DELETE=false
88
ADO_MANAGEMENT_POOL=private-pool-dev-uks
9+
CLOUD_API_HOSTNAME=review.manage-breast-screening.non-live.screening.nhs.uk

infrastructure/modules/arc-infra/azure_monitor.tf

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,3 +217,14 @@ resource "azurerm_resource_group_policy_remediation" "wac" {
217217
policy_assignment_id = azurerm_resource_group_policy_assignment.wac[0].id
218218
resource_discovery_mode = "ReEvaluateCompliance"
219219
}
220+
221+
module "app_insights" {
222+
count = var.enable_arc_servers ? 1 : 0
223+
source = "../dtos-devops-templates/infrastructure/modules/app-insights"
224+
225+
name = "ai-${var.app_short_name}-${var.env_config}-arc-uks"
226+
location = var.region
227+
resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name
228+
appinsights_type = "other"
229+
log_analytics_workspace_id = module.log_analytics_workspace[0].id
230+
}

infrastructure/modules/arc-infra/outputs.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ output "relay_namespace_hostname" {
2828
value = var.enable_arc_servers ? "${local.relay_namespace_name}.servicebus.windows.net" : null
2929
}
3030

31+
output "app_insights_connection_string" {
32+
description = "Application Insights connection string for the gateway services"
33+
sensitive = true
34+
value = var.enable_arc_servers ? module.app_insights[0].connection_string : null
35+
}
36+
3137
output "relay_listen_sas_keys" {
3238
description = "Per-machine relay listen SAS primary keys, keyed by Arc resource name. Used by the deploy pipeline to write .env files."
3339
sensitive = true

infrastructure/modules/arc-infra/relay.tf

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ resource "azurerm_relay_hybrid_connection" "per_machine" {
4545
requires_client_authorization = true
4646
}
4747

48-
# Listen-only SAS rule per HC — distributed to each gateway site via the deploy pipeline.
49-
# The cloud app uses a namespace-level Send SAS and does not need per-site keys.
48+
# Listen-only SAS rule per HC — retained for local development / break-glass access.
49+
# Production relay authentication uses Managed Identity (see relay_listener_role below).
5050
resource "azurerm_relay_hybrid_connection_authorization_rule" "per_machine_listen" {
5151
for_each = local.arc_machines
5252

@@ -59,3 +59,23 @@ resource "azurerm_relay_hybrid_connection_authorization_rule" "per_machine_liste
5959
send = false
6060
manage = false
6161
}
62+
63+
# Look up each discovered Arc machine to obtain its system-assigned managed identity.
64+
# Static machines (registered in the same apply) are excluded — they are not yet
65+
# visible to the data source and will be picked up on the next apply after onboarding.
66+
data "azurerm_arc_machine" "machines" {
67+
for_each = local.arc_machines_discovered
68+
name = each.key
69+
resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name
70+
}
71+
72+
# Grant each machine's MI the Azure Relay Listener role on its own HC so the relay
73+
# listener service can authenticate without a SAS key.
74+
module "relay_listener_role" {
75+
for_each = local.arc_machines_discovered
76+
source = "../dtos-devops-templates/infrastructure/modules/rbac-assignment"
77+
78+
scope = azurerm_relay_hybrid_connection.per_machine[each.key].id
79+
role_definition_name = "Azure Relay Listener"
80+
principal_id = data.azurerm_arc_machine.machines[each.key].identity[0].principal_id
81+
}

infrastructure/terraform/outputs.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
output "app_insights_connection_string" {
2+
description = "Application Insights connection string for the gateway services"
3+
sensitive = true
4+
value = module.arc_infra.app_insights_connection_string
5+
}
6+
17
output "relay_namespace_hostname" {
28
description = "Relay namespace FQDN for AZURE_RELAY_NAMESPACE in the gateway .env"
39
value = module.arc_infra.relay_namespace_hostname

infrastructure/terraform/resource_group_init/core.bicep

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ var roleID = {
3636
resourcePolicyContributor: '36243c78-bf99-498c-9df9-86d9f8d28608'
3737
virtualMachineAdministratorLogin: '1c0163c0-47e6-4577-8991-ea5c82e286e4'
3838
windowsAdminCenterAdministratorLogin: 'a6333a3e-0164-44c3-b281-7a577aff287f'
39+
azureRelayListener: '26e0b698-aa6d-4085-9386-aadae190014d'
3940
}
4041

4142
// Define role assignments for managed identity
@@ -61,7 +62,7 @@ var miRoleAssignments = [
6162
description: 'RBAC Administrator. Restricted to only assign/remove: Storage Blob Data Contributor, Storage Queue Data Contributor, Azure Connected Machine Onboarding, Azure Connected Machine Resource Administrator, Log Analytics Contributor, Virtual Machine Administrator Login, and Windows Admin Center Administrator Login.'
6263
// Delegated RBAC: This condition restricts the RBAC Administrator to only manage specific roles.
6364
// This is a security best practice that prevents the identity from granting itself or others sensitive roles like 'Owner' or 'User Access Administrator'.
64-
condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}}))'
65+
condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}}))'
6566
conditionVersion: '2.0'
6667
}
6768
]
@@ -84,7 +85,7 @@ var groupRoleAssignments = [
8485
description: 'RBAC Administrator. Restricted to only assign/remove: Storage Blob Data Contributor, Storage Queue Data Contributor, Azure Connected Machine Onboarding, Azure Connected Machine Resource Administrator, Log Analytics Contributor, Virtual Machine Administrator Login, and Windows Admin Center Administrator Login.'
8586
// Delegated RBAC: This condition restricts the RBAC Administrator to only manage specific roles.
8687
// This is a security best practice that prevents the identity from granting itself or others sensitive roles like 'Owner' or 'User Access Administrator'.
87-
condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}}))'
88+
condition: '((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/write\'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}})) AND ((!(ActionMatches{\'Microsoft.Authorization/roleAssignments/delete\'})) OR (@Resource[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {${roleID.storageBlobDataContributor}, ${roleID.storageQueueDataContributor}, ${roleID.AzureConnectedMachineOnboarding}, ${roleID.AzureConnectedMachineResourceAdministrator}, ${roleID.logAnalyticsContributor}, ${roleID.virtualMachineAdministratorLogin}, ${roleID.windowsAdminCenterAdministratorLogin}, ${roleID.azureRelayListener}}))'
8889
conditionVersion: '2.0'
8990
}
9091
]

scripts/bash/deploy_arc_ring.sh

Lines changed: 14 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,38 +2,36 @@
22
# Deploy the gateway app to all Arc machines matching a ring within an environment.
33
# Called by deploy_stage.sh.
44
#
5-
# Usage: deploy_arc_ring.sh <environment> <ring> <release_tag> <kv_name>
5+
# Usage: deploy_arc_ring.sh <environment> <ring> <release_tag>
66

77
set -euo pipefail
88

99
ENVIRONMENT=$1
1010
RING=$2
1111
RELEASE_TAG=$3
12-
KV_NAME=$4
1312

1413
APP_SHORT_NAME="mbsgw"
1514
ARC_RG="rg-${APP_SHORT_NAME}-${ENVIRONMENT}-uks-arc-enabled-servers"
1615

17-
# Relay namespace is owned by dtos-manage-breast-screening; derive from environment name.
18-
RELAY_NAMESPACE_NAME="relay-manbrs-${ENVIRONMENT}"
19-
RELAY_RG="rg-manbrs-${ENVIRONMENT}-uks"
20-
RELAY_NAMESPACE_HOSTNAME="${RELAY_NAMESPACE_NAME}.servicebus.windows.net"
21-
22-
# Ensure the relay extension is installed
23-
if ! az relay --help &>/dev/null; then
24-
echo "Installing Azure CLI 'relay' extension..."
25-
az extension add --name relay || {
26-
echo "ERROR: Failed to install 'relay' extension. Please run 'az extension add --name relay' manually."
27-
exit 1
28-
}
29-
fi
16+
RELAY_NAMESPACE_HOSTNAME="relay-manbrs-${ENVIRONMENT}.servicebus.windows.net"
3017

3118
# Use forward slashes — Python handles these fine on Windows and avoids .env escaping issues
3219
BASE_PATH="C:/Program Files/NHS/ManageBreastScreeningGateway"
3320
PYTHON_VERSION=$(awk '/^python / {print $2}' .tool-versions)
3421

3522
echo "--- Ring: ${RING} | Environment: ${ENVIRONMENT} | Release: ${RELEASE_TAG} ---"
3623

24+
# ── Per-environment config ─────────────────────────────────────────────────────
25+
source "infrastructure/environments/${ENVIRONMENT}/variables.sh"
26+
CLOUD_API_ENDPOINT="https://${CLOUD_API_HOSTNAME}/api/v1/dicom"
27+
28+
APPLICATIONINSIGHTS_CONNECTION_STRING=$(az monitor app-insights component show \
29+
--app "ai-${APP_SHORT_NAME}-${ENVIRONMENT}-arc-uks" \
30+
--resource-group "$ARC_RG" \
31+
--query connectionString -o tsv 2>/dev/null || echo "")
32+
[[ -z "$APPLICATIONINSIGHTS_CONNECTION_STRING" ]] && \
33+
echo "##vso[task.logissue type=warning]Application Insights resource not found — telemetry will be disabled"
34+
3735
# ── Discover machines ──────────────────────────────────────────────────────────
3836
MACHINES_JSON=$(az connectedmachine list \
3937
--resource-group "$ARC_RG" \
@@ -61,44 +59,13 @@ while IFS= read -r MACHINE_JSON; do
6159
LOCATION=$(echo "$MACHINE_JSON" | jq -r '.location')
6260
echo "Preparing deploy for $MACHINE ($LOCATION)..."
6361

64-
# Fetch relay SAS key directly — Contributor includes listKeys on relay HCs,
65-
# and this avoids any dependency on Terraform state having the resource imported.
66-
echo "Fetching SAS key for hc-${MACHINE} in $RELAY_NAMESPACE_NAME..."
67-
SAS_KEY=$(az relay hyco authorization-rule keys list \
68-
--resource-group "$RELAY_RG" \
69-
--namespace-name "$RELAY_NAMESPACE_NAME" \
70-
--hybrid-connection-name "hc-${MACHINE}" \
71-
--name listen \
72-
--query primaryKey -o tsv 2>/tmp/relay_key_err_${MACHINE}) || {
73-
ERR=$(cat /tmp/relay_key_err_${MACHINE})
74-
echo "##vso[task.logissue type=warning]Failed to fetch relay SAS key for hc-${MACHINE}: $ERR"
75-
SAS_KEY=""
76-
}
77-
78-
[[ -z "$SAS_KEY" ]] && \
79-
echo "##vso[task.logissue type=warning]No relay SAS key found for hc-${MACHINE} — relay listener will not connect"
80-
81-
# Cloud API secrets are optional — warn if absent, services still start
82-
CLOUD_API_ENDPOINT=$(az keyvault secret show --vault-name "$KV_NAME" \
83-
--name "cloud-api-endpoint" --query value -o tsv 2>/dev/null || echo "")
84-
CLOUD_API_TOKEN=$(az keyvault secret show --vault-name "$KV_NAME" \
85-
--name "cloud-api-token-${MACHINE}" --query value -o tsv 2>/dev/null || echo "")
86-
87-
[[ -z "$CLOUD_API_ENDPOINT" ]] && \
88-
echo "##vso[task.logissue type=warning]cloud-api-endpoint not in $KV_NAME — Upload service will not reach cloud API for $MACHINE"
89-
[[ -z "$CLOUD_API_TOKEN" ]] && \
90-
echo "##vso[task.logissue type=warning]cloud-api-token-${MACHINE} not in $KV_NAME — Upload service will not authenticate for $MACHINE"
91-
9262
# Build .env, then base64-encode to pass newlines as a run command parameter.
9363
# NOTE: Arc Run Command drops protectedParameters for inline source.script,
9464
# so EnvContentB64 travels as a regular parameter (base64-encoded, not plain text).
95-
# TODO: migrate to Key Vault + Arc MSI for production environments.
9665
ENV_CONTENT="AZURE_RELAY_NAMESPACE=${RELAY_NAMESPACE_HOSTNAME}
9766
AZURE_RELAY_HYBRID_CONNECTION=hc-${MACHINE}
98-
AZURE_RELAY_KEY_NAME=listen
99-
AZURE_RELAY_SHARED_ACCESS_KEY=${SAS_KEY}
10067
CLOUD_API_ENDPOINT=${CLOUD_API_ENDPOINT}
101-
CLOUD_API_TOKEN=${CLOUD_API_TOKEN}
68+
APPLICATIONINSIGHTS_CONNECTION_STRING=${APPLICATIONINSIGHTS_CONNECTION_STRING}
10269
MWL_AET=SCREENING_MWL
10370
MWL_PORT=4243
10471
MWL_DB_PATH=${BASE_PATH}/data/worklist.db

0 commit comments

Comments
 (0)