diff --git a/.azuredevops/pipelines/deploy-app.yml b/.azuredevops/pipelines/deploy-app.yml index 898877a6..d47266ba 100644 --- a/.azuredevops/pipelines/deploy-app.yml +++ b/.azuredevops/pipelines/deploy-app.yml @@ -36,7 +36,7 @@ stages: - task: AzureCLI@2 displayName: Deploy gateway app to ${{ parameters.environment }} inputs: - azureSubscription: manbgw-${{ parameters.environment }} + azureSubscription: mbsgw-${{ parameters.environment }} scriptType: bash scriptLocation: inlineScript addSpnToEnvironment: true diff --git a/.azuredevops/pipelines/deploy.yml b/.azuredevops/pipelines/deploy.yml index 51e79a4a..4dc3c45e 100644 --- a/.azuredevops/pipelines/deploy.yml +++ b/.azuredevops/pipelines/deploy.yml @@ -43,7 +43,7 @@ stages: - task: AzureCLI@2 displayName: Run Terraform inputs: - azureSubscription: manbgw-${{ parameters.environment }} + azureSubscription: mbsgw-${{ parameters.environment }} scriptType: bash scriptLocation: inlineScript addSpnToEnvironment: true diff --git a/.github/workflows/cicd-2-main-branch.yaml b/.github/workflows/cicd-2-main-branch.yaml index e09da2cc..686a7188 100644 --- a/.github/workflows/cicd-2-main-branch.yaml +++ b/.github/workflows/cicd-2-main-branch.yaml @@ -104,7 +104,7 @@ jobs: id-token: write uses: ./.github/workflows/stage-4-deploy.yaml with: - environments: '["review"]' + environments: '["review", "dev", "preprod"]' commit_sha: ${{ github.sha }} secrets: inherit @@ -120,7 +120,7 @@ jobs: id-token: write uses: ./.github/workflows/stage-4-deploy-app.yaml with: - environments: '["dev", "preprod", "prod"]' + environments: '["dev", "preprod"]' release_tag: ${{ needs.release-stage.outputs.new_release_version }} commit_sha: ${{ github.sha }} secrets: inherit diff --git a/docs/deployment/16-Deploy-Script-Analysis.md b/docs/deployment/16-Deploy-Script-Analysis.md deleted file mode 100644 index 2bbb9a75..00000000 --- a/docs/deployment/16-Deploy-Script-Analysis.md +++ /dev/null @@ -1,280 +0,0 @@ -# 16. Deploy Script — Step-by-Step Analysis - -> **System**: manage-breast-screening-gateway -> **Script**: `scripts/powershell/deploy.ps1` -> **Strategy**: Blue/Green deployment with automatic rollback -> **Cross-references**: [Doc 15 - Deployment & Rollback Runbook](./15-Deployment-Rollback-Runbook.md) | [Doc 7 - DR/Backup](./7-DR-Backup-Strategy.md) - ---- - -## Overview - -The deploy script automates the full lifecycle of deploying the Manage Breast Screening Gateway onto a Windows Server VM. By default, it downloads the release package directly from GitHub Releases, with an override to use a local file instead. It follows a blue/green deployment model where a new release is prepared in isolation, services are atomically switched via a filesystem junction, and an automatic rollback is triggered if any service fails to start or crashes shortly after launch. - ---- - -## Step-by-Step Breakdown - -### Step 1: Logging Configuration - -```text -Lines 49–68 -``` - -**What it does**: Creates a `logs\deployments` directory under the install path and opens a timestamped log file. All subsequent operations write to both the console (colour-coded) and this file. - -**Why it matters**: Every deployment leaves a persistent audit trail. If a deployment fails at 2 AM and the operator investigates the next morning, the log file contains the exact sequence of events with millisecond timestamps. The dual-output approach (file + console) means nothing is lost if the terminal session disconnects. - ---- - -### Step 2: Helper Functions - -```text -Lines 70–100 -``` - -**What it does**: Defines internal helpers: -- `Invoke-Nssm` — Wraps every NSSM call and checks `$LASTEXITCODE`. Throws immediately with the full command string if NSSM returns a non-zero exit code. -- `Stop-AllServices` — Gracefully stops all components with individual timeout enforcement. - -**Why it matters**: NSSM is a native executable and PowerShell does not treat its non-zero exit codes as errors by default. Without explicit checking, a failed `nssm install` or `nssm set` would silently proceed, creating a service that is misconfigured. The helper eliminates this class of bug across all ~12 NSSM invocations. - ---- - -### Step 3: Bootstrap (Optional) - -```text -Lines 165–206 -``` - -**What it does**: When `-Bootstrap` is passed, installs Chocolatey (if absent), Python, uv, and NSSM. Each tool is only installed if not already present in PATH. The Python version comes from the `-PythonVersion` parameter (sourced from `.tool-versions` in the repo via the deploy pipeline), with a fallback to `3.14.0` if the parameter is not provided. - -**Why it matters**: -- **Idempotent**: Running bootstrap twice does not reinstall tools that are already present (including Python). This makes the script safe to re-run after a partial failure without triggering Chocolatey "already installed" errors. -- **Single source of truth**: The Python version is always passed from `.tool-versions` at the root of the repo, read by `deploy_arc_ring.sh` at deploy time. This ensures the VM always runs the same Python version as CI. - ---- - -### Step 4: Package Acquisition (GitHub Release Download) - -```text -Lines 208–258 -``` - -**What it does**: If no `-ZipPath` is provided, the script downloads the release package from GitHub Releases. It queries the GitHub API for the specified release tag (or `latest` by default), locates the `gateway-*.zip` asset and its `.sha256` checksum, and downloads both to a local `downloads` directory under the install path. When `-ZipPath` is provided, this step is skipped entirely. - -**Why it matters**: -- **Zero-touch default**: Operators can deploy with a single command (`.\deploy.ps1`) without manually downloading or transferring files. This removes a common source of human error — wrong version, corrupted transfer, or stale artifact. -- **Private repo support**: The optional `-GitHubToken` parameter adds a `Bearer` token to API requests, enabling downloads from private repositories without installing the `gh` CLI on the VM. -- **No external tooling**: Uses `Invoke-RestMethod` and `Invoke-WebRequest`, both built into PowerShell 5.1. No `gh` CLI, `curl`, or `wget` required on the target machine. -- **TLS enforcement**: Explicitly enables TLS 1.2+ before the API call, which is necessary on older Windows Server builds where PowerShell defaults to TLS 1.0. -- **Override escape hatch**: The `-ZipPath` parameter completely bypasses downloading, supporting air-gapped environments, manual testing, or CI pipelines that pre-stage the artifact. - ---- - -### Step 5: Prerequisite Validation - -```text -Lines 260–282 -``` - -**What it does**: Verifies that `python.exe`, `uv.exe`, and `nssm.exe` are in PATH, and that `ZipPath` points to an existing file (whether downloaded or provided). If a `.sha256` sidecar file exists, verifies the archive's SHA256 hash. - -**Why it matters**: -- **Uses `throw` instead of `exit 1`**: The script uses `throw` for all failures, which is safer than `exit 1`. When a PowerShell script using `exit` is dot-sourced or called from another script, `exit` terminates the entire host process. `throw` propagates an error that the caller can catch with `try/catch`, making the script composable. -- **SHA256 verification**: Catches corrupted downloads or tampered archives before any extraction occurs. This is a defense-in-depth measure against supply chain issues. - ---- - -### Step 6: Directory Structure Preparation - -```text -Lines 284–302 -``` - -**What it does**: Ensures the `releases`, `data`, and `logs` directories exist under the base install path. - -**Why it matters**: The `data` directory persists across deployments (it is never inside a release folder), so application databases and state survive upgrades. The separation of `releases` (immutable, versioned) from `data` (mutable, persistent) is a core tenet of the blue/green model. - ---- - -### Step 7: Package Extraction - -```text -Lines 304–367 -``` - -**What it does**: -1. Extracts the outer ZIP to a temporary staging directory. -2. Looks for an inner ZIP. If found, verifies its SHA256 (if a checksum file is present), then extracts it to the version directory. -3. Falls back to treating the outer ZIP as the application if no inner ZIP exists. -4. Handles nested folder structures (single top-level directory inside the archive). -5. Validates that `pyproject.toml` and `uv.lock` are present in the extracted package. - -**Why it matters**: -- **Staging directory**: Extraction happens in `%TEMP%`, not in the install path. If extraction fails, the install directory is untouched. The staging directory is cleaned up in a `finally` block, so it never leaks even on error. -- **Locked file handling**: When re-deploying the same version, the previous release directory may contain locked `.pyd` (native DLL) files from Python packages. The script clears file attributes before removal, preventing "Access denied" errors on compiled extension modules. -- **Double integrity verification**: Both the outer wrapper and the inner application archive can be checksummed independently. This catches corruption at the transport layer and at the packaging layer. -- **`uv.lock` validation**: The subsequent `uv sync --frozen` command requires a lockfile. Without this check, the script would fail with a confusing uv error message deep in the dependency resolution phase. Validating early gives a clear, actionable error. - ---- - -### Step 8: Virtual Environment Setup - -```text -Lines 368–390 -``` - -**What it does**: Creates a Python virtual environment inside the version directory using `uv venv`, then installs dependencies with `uv sync --frozen`. After dependency installation, pre-compiles the Python bytecache for both the application source and the standard library. Both external commands check `$LASTEXITCODE` and throw on failure. - -**Why it matters**: -- **Isolated per release**: Each release gets its own `.venv`. The previous release's environment is never modified. This is what makes rollback possible — the old environment is still intact. -- **Frozen sync**: `--frozen` means uv installs exactly what is in `uv.lock`, with no resolution. This guarantees the deployed dependencies are identical to what CI tested. No "works on my machine" drift. -- **Bytecache pre-compilation**: Python 3.14 compiles `.pyc` files for the standard library on first import. Without pre-compilation, this happens when services start under NSSM, taking long enough for NSSM to consider the service hung. Running `compileall` during deployment eliminates this first-run penalty. -- **Error propagation**: The `catch` block logs the error and re-throws (not `exit 1`), preserving the call stack for the caller. - ---- - -### Step 9: Service Helper Generation - -```text -Lines 392–404 -``` - -**What it does**: Generates `.bat` wrapper scripts for each of the four Gateway services (Relay, PACS, MWL, Upload). Each script sets the working directory, configures `PYTHONPATH=src`, and launches the Python entry point via the release's own virtual environment. - -**Why it matters**: -- **`PYTHONPATH=src`**: The entry points (e.g., `pacs_main.py`) import from sibling modules in `src/`. Without setting `PYTHONPATH`, Python cannot resolve these imports when running from the release root directory. This mirrors how the CI smoke test runs. -- **Location-independent**: By using `%~dp0` (the directory of the batch file itself), the scripts work whether accessed via the junction or directly. -- **Isolated environment**: The `.venv\Scripts\python.exe` path ensures the correct per-release virtual environment is always used. - ---- - -### Step 10: Capture Previous Version for Rollback - -```text -Lines 411–420 -``` - -**What it does**: Before any destructive action, reads the current junction's target path (if it exists) and stores it in `$previousVersionDir`. - -**Why it matters**: This is the safety net. If anything goes wrong during cutover, the script knows exactly where the last known-good release lives. Without this, a failed deployment would leave the system with no path back to a working state. - ---- - -### Step 11: Service Stop with Timeout - -```text -Lines 422–423 -``` - -**What it does**: Calls `Stop-AllServices` to gracefully stop each component with individual timeout enforcement. If a timeout is exceeded, the deployment aborts before touching the junction. - -**Why it matters**: -- **Graceful shutdown**: Sends a stop signal and gives the service time to finish in-flight DICOM transfers. -- **Abort before damage**: The junction has not been modified yet at this point. If stop fails, the system is still running the old version and no rollback is needed — the deployment simply aborts cleanly. - ---- - -### Step 12: Junction Swap (Atomic Cutover) - -```text -Lines 457–458 -``` - -**What it does**: Deletes the old `current` junction and creates a new one pointing to the new version directory. - -**Why it matters**: This is the atomic switch. From the perspective of NSSM and all service configurations, `current` is a stable path that never changes — only its target changes. This means services do not need to be reconfigured with new absolute paths on every deployment. - ---- - -### Step 13: Service Registration and Start - -```text -Lines 460–495 -``` - -**What it does**: For each service, removes any existing registration and creates a fresh one with NSSM. Configures stdout/stderr logging. Starts each service and tracks which ones were started successfully. - -**Why it matters**: -- **Clean registration**: Rather than trying to update existing services in-place, the script removes and re-registers each service on every deployment. This eliminates NSSM's internal "recently crashed" throttle counter that persists from previous failed deployments. -- **SCM synchronisation (Race Condition Protection)**: After removing a service, the script enters a polling loop that queries the Windows Service Control Manager (SCM) until the service is fully deregistered. In Windows, service deletion is asynchronous; attempting to `install` a service immediately after `remove` often fails with "service marked for deletion." This loop guarantees the SCM is ready before the next step. -- **Tracked starts**: `$startedServices` records which services were started. If a failure occurs partway through, only the services that were started are stopped during rollback — avoiding errors from trying to stop services that were never started. -- **Break on first failure**: If any service fails to start, the loop breaks immediately rather than continuing to start services that depend on a broken state. - ---- - -### Step 14: Post-Start Health Checks - -```text -Lines 497–519 -``` - -**What it does**: After all services start, polls each one repeatedly (default: 5 attempts, 2s apart) to verify it remains in `Running` state. If a service crashes (e.g., import error, missing config), the health check detects it. - -**Why it matters**: `Start-Service` succeeding only means the service control manager accepted the start request. The actual process could crash within milliseconds. The health check window catches: -- Missing Python dependencies despite `uv sync` -- Configuration errors (missing `.env` variables) -- Port conflicts with another process -- Import errors in the application code - -Without this check, a deployment that appears successful could leave the gateway fully down. - ---- - -### Step 15: Automatic Rollback - -```text -Lines 521–551 -``` - -**What it does**: If either service start or health check failed: -1. Stops all services started in the failed attempt -2. Re-points the junction to the previous release directory -3. Updates NSSM paths to point through the restored junction -4. Restarts all services with the previous version -5. Throws an error so the caller knows the deployment failed - -**Why it matters**: This is the core production-readiness guarantee. In a hospital environment, a non-functional gateway means mammography images cannot be transferred. Automatic rollback minimises the window of unavailability from "until an operator notices and acts" to "a few seconds while the script restores the previous version." - -If no previous version exists (first-ever deployment), the script logs a clear warning instead of crashing during rollback. - ---- - -### Step 16: Release Cleanup & Deferred Trash - -```text -Lines 425–454 -``` - -**What it does**: Before switching the junction, the script identifies old releases to prune (keeping the latest 3). If a directory cannot be deleted because a file is locked (common with `.pyd` native modules or antivirus scans), the script **moves** the directory to a `.trash-` folder. On the *next* deployment, the script first attempts to empty all `.trash` folders. - -**Why it matters**: -- **Zero-fail pruning**: Windows filesystem locks often cause "Access Denied" during recursive deletes. By moving locked folders to a "trash" area, the script ensures the primary `releases` directory stays clean and the deployment continues even if an old version is stubborn. -- **Disk management**: Disk space on hospital VMs is finite. Each release contains a full `.venv` which can be hundreds of megabytes. Keeping 3 releases provides: - - The current running version - - One previous version (for manual rollback investigation) - - One additional version for safety margin -- **Strategic timing**: Cleanup happens while services are stopped but before new ones start, maximizing the chance that file locks have been released. - ---- - -## Reliability Properties Summary - -| Property | How the Script Achieves It | -|----------|----------------------------| -| **Atomic cutover** | Filesystem junction swap — services always reference `current\`, only the target changes | -| **Automatic rollback** | Previous junction target is captured before cutover; restored if any service fails start or health check | -| **Automated acquisition** | Downloads from GitHub Releases by default; no manual file transfer needed. Supports private repos via token. `-ZipPath` override for air-gapped environments | -| **Integrity verification** | SHA256 checksums verified at both outer and inner archive levels | -| **Bytecache pre-compilation** | Runs `compileall` on source and stdlib after `uv sync`, eliminating Python 3.14's slow first-run `.pyc` compilation that causes NSSM to consider services hung | -| **Idempotent** | Bootstrap skips installed tools (including Python); service registration does clean remove + reinstall to clear NSSM throttle state | -| **Fail-fast** | All validation (prerequisites, archive integrity, `pyproject.toml`, `uv.lock`) happens before any destructive action | -| **Reproducible builds** | `uv sync --frozen` installs exact lockfile versions — no resolution, no drift | -| **Isolated releases** | Each version has its own `.venv` — old releases are never modified, enabling clean rollback | -| **Auditable** | Every operation is written to a timestamped log file with millisecond precision | -| **Composable** | Uses `throw` instead of `exit 1` — safe to call from other scripts, pipelines, or orchestrators | -| **Timeout-protected** | Service stops have configurable deadlines; deployment aborts before cutover if a service won't stop | -| **External tool safety** | Every NSSM invocation checks `$LASTEXITCODE` via `Invoke-Nssm` helper; SCM polling prevents race conditions during service re-installation | -| **Cleanup** | Staging directories are cleaned in `finally` blocks; old releases are pruned with a **deferred trash** system to handle Windows file locks | -| **ASCII-safe** | All script content is pure ASCII, preventing encoding corruption when files are transferred to Windows VMs with non-UTF-8 default encodings | diff --git a/docs/deployment/infrastructure/create-environment.md b/docs/deployment/infrastructure/create-environment.md new file mode 100644 index 00000000..aa6d7a65 --- /dev/null +++ b/docs/deployment/infrastructure/create-environment.md @@ -0,0 +1,235 @@ +# Create an environment + +This is the initial manual process to create a new Azure environment (e.g. `dev`, `preprod`, `prod`). + +All commands assume you are authenticated to Azure via `az login` and are running from the root of this repository unless stated otherwise. + +## Prerequisites + +- Owner role on both the hub and spoke Azure subscriptions +- Access to Azure DevOps project `manage-breast-screening-gateway` at `https://dev.azure.com/nhse-dtos` +- Access to the GitHub repository `NHSDigital/manage-breast-screening-gateway` +- Access to an Azure Virtual Desktop (AVD) session connected to the internal network + +## Azure resource providers + +The following resource providers must be registered on the **spoke subscription** before running Terraform. Missing registrations will cause obscure errors during `terraform apply`. + +Run these once per subscription: + +```bash +az provider register --namespace Microsoft.HybridCompute +az provider register --namespace Microsoft.GuestConfiguration +az provider register --namespace Microsoft.HybridConnectivity +az provider register --namespace Microsoft.Relay +az provider register --namespace Microsoft.PolicyInsights +az provider register --namespace Microsoft.Insights +az provider register --namespace Microsoft.OperationalInsights +az provider register --namespace Microsoft.ManagedIdentity +``` + +Verify registration status (wait until all show `Registered`): + +```bash +az provider show --namespace Microsoft.HybridCompute --query registrationState -o tsv +az provider show --namespace Microsoft.HybridConnectivity --query registrationState -o tsv +az provider show --namespace Microsoft.GuestConfiguration --query registrationState -o tsv +``` + +## Entra ID + +- Create Entra ID group in the `Digital screening` Administrative Unit: + - `screening_mbsgw_[environment]` + +- Ask CCOE to assign the following roles to `mi-mbsgw-[environment]-adotoaz-uks`: + - [Form for PIM](https://nhsdigitallive.service-now.com/nhs_digital?id=sc_cat_item&sys_id=28f3ab4f1bf3ca1078ac4337b04bcb78&sysparm_category=114fced51bdae1502eee65b9bd4bcbdc) + - Role Names: `Group.Read.All` and `Directory Readers` + - Application Name: `mi-mbsgw-[environment]-adotoaz-uks` + - Managed Identity: `mi-mbsgw-[environment]-adotoaz-uks` + - Description: Required to resolve the Arc onboarding service principal by display name during Terraform runs + +- Confirm the Arc onboarding service principal exists: + - Name: `spn-azure-arc-onboarding-screening-[environment]` + - If it does not exist, raise a request with the platform team to create it + +## Code + +- Create the configuration files in `infrastructure/environments/[environment]/`: + - `variables.sh` — Azure subscription names, hub subscription, ADO pool, soft delete flag + - `variables.tfvars` — Terraform variable overrides (can be empty if all defaults apply) +- Add an `[environment]:` target in `scripts/terraform/terraform.mk` following the existing pattern +- Add `[environment]` to the `environments` list in the `deploy-stage` step of `.github/workflows/cicd-2-main-branch.yaml` + +## Bicep (resource group initialisation) + +> [!IMPORTANT] +> This step creates the Azure infrastructure that Terraform depends on — managed identities, Terraform state storage, Key Vault, and resource groups. It must be completed before running Terraform. +> **Required**: Owner role on both the hub and spoke subscriptions. + +From an AVD session: + +1. Login with Microsoft Graph scope: + + ```bash + az login --scope https://graph.microsoft.com//.default -t HSCIC365.onmicrosoft.com + ``` + +2. Run the Bicep deployment: + + ```bash + make [environment] resource-group-init + ``` + +This deploys two Bicep templates: + +**Hub subscription** (`main.bicep`) creates: + +- Managed identity `mi-mbsgw-[environment]-adotoaz-uks` — used by ADO to deploy Azure resources via Terraform +- Managed identity `mi-mbsgw-[environment]-ghtoado-uks` — used by GitHub Actions to trigger ADO pipelines (OIDC federated) +- Terraform state storage account `sambsgw[environment]tfstate` with private endpoint +- Infra Key Vault `kv-mbsgw-[environment]-inf` with private endpoint +- Reader role assigned to `mi-mbsgw-[environment]-ghtoado-uks` at hub subscription scope + +**Spoke subscription** (`core.bicep`) creates: + +- Resource group `rg-mbsgw-[environment]-uks-arc-enabled-servers` — where Arc machines register +- Contributor, RBAC Administrator (delegated), and Resource Policy Contributor roles for `mi-mbsgw-[environment]-adotoaz-uks` +- Contributor and Key Vault Secrets Officer roles for the `screening_mbsgw_[environment]` Entra ID group + +Note the outputs printed at the end of the deployment — you will need the managed identity client IDs in subsequent steps. + +## Azure DevOps + +### ADO group + +- Navigate to **Project Settings → Permissions → New group** +- Name: `Run pipeline - [environment]` +- Members: `mi-mbsgw-[environment]-ghtoado-uks` + - There may be more than one identity with a similar name — check the client ID printed below the name matches the one from the Bicep output +- Permissions (at project level): + - View project-level information + +### ADO pipelines + +Create two pipelines, applying the same security settings to each: + +**Infrastructure pipeline** — deploys Arc infrastructure via Terraform: + +- Navigate to **Pipelines → New pipeline** +- Source: Azure Repos Git → select this repository +- Existing Azure Pipelines YAML file: `.azuredevops/pipelines/deploy.yml` +- Name: `Deploy to Azure - [environment]` +- Do not run yet + +**App pipeline** — deploys the gateway app to Arc machines: + +- Navigate to **Pipelines → New pipeline** +- Source: Azure Repos Git → select this repository +- Existing Azure Pipelines YAML file: `.azuredevops/pipelines/deploy-app.yml` +- Name: `Deploy Gateway App - [environment]` +- Do not run yet + +For each pipeline, manage security: + +- Navigate to the pipeline → **⋮ → Manage security** +- Add group: `Run pipeline - [environment]` +- Grant permissions: + - Edit queue build configuration + - Queue builds + - View build pipeline + - View builds + +### ADO service connection + +> [!NOTE] +> If the Managed Identity dropdown is empty (common with guest accounts), use the **create manually** link and enter the values directly. + +- Navigate to **Project Settings → Service connections → New service connection** +- Connection type: `Azure Resource Manager` +- Authentication method: `Workload Identity Federation (manual)` +- Issuer: `https://token.actions.githubusercontent.com` +- Subject identifier: `repo:NHSDigital/manage-breast-screening-gateway:environment:[environment]` +- Enter the following (from the Bicep output or by running the commands below): + +```bash +# Client ID and tenant ID of mi-mbsgw-[environment]-adotoaz-uks +az identity show --name mi-mbsgw-[environment]-adotoaz-uks \ + --resource-group rg-mi-[environment]-uks \ + --subscription "[hub-subscription-name]" \ + --query "{clientId: clientId, tenantId: tenantId}" -o json + +# Spoke subscription ID +az account show --subscription "[spoke-subscription-name]" --query id -o tsv +``` + +- Scope level: `Subscription` +- Subscription ID: `[spoke-subscription-id]` +- Subscription name: `[spoke-subscription-name]` +- Resource group for service connection: leave blank +- Service connection name: `mbsgw-[environment]` +- Do **NOT** tick: Grant access permission to all pipelines + +Manage service connection security: + +- Navigate to the service connection → **⋮ → Security** +- Add both `Deploy to Azure - [environment]` and `Deploy Gateway App - [environment]` pipelines with permission to use the connection + +### ADO environment + +- Navigate to **Pipelines → Environments → New environment** +- Name: `[environment]` +- Resource: None +- Set exclusive lock (all environments except `review`) +- Add pipeline permission for both `Deploy to Azure - [environment]` and `Deploy Gateway App - [environment]` + +## GitHub + +- Create a GitHub environment named `[environment]`: + - Navigate to **Settings → Environments → New environment** + +- Add protection rules (all environments except `review`): + - Deselect `Allow administrators to bypass configured protection rules` + - In `Deployment branches and tags` choose `Selected branches and tags` + - Click `Add deployment branch or tag rule` and enter `main` + +- Add the following environment secrets (values from `mi-mbsgw-[environment]-ghtoado-uks`): + +```bash +# AZURE_CLIENT_ID — client ID of mi-mbsgw-[environment]-ghtoado-uks +az identity show --name mi-mbsgw-[environment]-ghtoado-uks \ + --resource-group rg-mi-[environment]-uks \ + --subscription "[hub-subscription-name]" \ + --query clientId -o tsv + +# AZURE_TENANT_ID +az account show --query tenantId -o tsv + +# AZURE_SUBSCRIPTION_ID — hub subscription ID (where the managed identity has Reader) +az account show --subscription "[hub-subscription-name]" --query id -o tsv +``` + +| Secret | Value | +| ---------------------- | ------------------------------------------------- | +| `AZURE_CLIENT_ID` | Client ID of `mi-mbsgw-[environment]-ghtoado-uks` | +| `AZURE_TENANT_ID` | Azure tenant ID | +| `AZURE_SUBSCRIPTION_ID` | Hub subscription ID | + +## First run + +1. Merge a pull request to `main` to trigger the CI/CD pipeline, or trigger the ADO pipeline manually from Azure DevOps +2. On the first run you will be prompted to authorise in ADO: + - Service connection access by the pipeline + - Agent pool access by the environment +3. Check the pipeline completes successfully — Terraform will create: + - Arc-enabled server RBAC assignments + - Azure Relay hybrid connections (empty on first deploy — populated as Arc machines onboard) + - Log Analytics workspace and Data Collection Rule + - Azure Monitor policy assignments and remediation tasks + +## Arc machine discovery + +Terraform automatically discovers Arc machines registered in the Arc-enabled servers resource group and creates a relay Hybrid Connection for each one. On first deploy the resource group is empty, so no connections are created. After each Arc machine is onboarded at a hospital site, run `terraform apply` to pick it up: + +```bash +make [environment] terraform-apply +``` diff --git a/infrastructure/environments/dev/variables.tfvars b/infrastructure/environments/dev/variables.tfvars index e69de29b..a1042038 100644 --- a/infrastructure/environments/dev/variables.tfvars +++ b/infrastructure/environments/dev/variables.tfvars @@ -0,0 +1 @@ +enable_gateway_test_vm = true diff --git a/infrastructure/environments/review/variables.tfvars b/infrastructure/environments/review/variables.tfvars index bb097c1f..a1042038 100644 --- a/infrastructure/environments/review/variables.tfvars +++ b/infrastructure/environments/review/variables.tfvars @@ -1,2 +1 @@ -vnet_address_space = "10.131.0.0/16" enable_gateway_test_vm = true diff --git a/infrastructure/modules/arc-infra/arc.tf b/infrastructure/modules/arc-infra/arc.tf index d926b030..8bb0ab33 100644 --- a/infrastructure/modules/arc-infra/arc.tf +++ b/infrastructure/modules/arc-infra/arc.tf @@ -1,9 +1,9 @@ -# Resource group for Arc-enabled servers (where machines register on onboarding) -resource "azurerm_resource_group" "arc_enabled_servers" { +# Resource group for Arc-enabled servers — created by resource-group-init (Bicep), +# not managed by Terraform, so it exists before this module runs. +data "azurerm_resource_group" "arc_enabled_servers" { count = var.enable_arc_servers ? 1 : 0 - name = "${var.resource_group_name}-arc-enabled-servers" - location = var.region + name = "${var.resource_group_name}-arc-enabled-servers" } # Look up the Arc onboarding service principal by its standard name @@ -18,7 +18,7 @@ module "arc_onboarding_role" { source = "../dtos-devops-templates/infrastructure/modules/rbac-assignment" - scope = azurerm_resource_group.arc_enabled_servers[0].id + scope = data.azurerm_resource_group.arc_enabled_servers[0].id role_definition_name = "Azure Connected Machine Onboarding" principal_id = data.azuread_service_principal.arc_onboarding[0].object_id } diff --git a/infrastructure/modules/arc-infra/azure_monitor.tf b/infrastructure/modules/arc-infra/azure_monitor.tf index 49e2f44b..35a9fe54 100644 --- a/infrastructure/modules/arc-infra/azure_monitor.tf +++ b/infrastructure/modules/arc-infra/azure_monitor.tf @@ -5,7 +5,7 @@ module "log_analytics_workspace" { name = "law-${var.app_short_name}-${var.env_config}-arc-uks" location = var.region - resource_group_name = azurerm_resource_group.arc_enabled_servers[0].name + resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name law_sku = "PerGB2018" retention_days = 30 @@ -19,7 +19,7 @@ resource "azurerm_monitor_data_collection_rule" "arc" { name = "dcr-${var.app_short_name}-${var.env_config}-arc-uks" location = var.region - resource_group_name = azurerm_resource_group.arc_enabled_servers[0].name + resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name destinations { log_analytics { @@ -65,7 +65,7 @@ module "arc_monitor_policy_identity" { source = "../dtos-devops-templates/infrastructure/modules/managed-identity" uai_name = "mi-${var.app_short_name}-${var.env_config}-arc-monitor-uks" - resource_group_name = azurerm_resource_group.arc_enabled_servers[0].name + resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name location = var.region } @@ -74,21 +74,31 @@ module "arc_monitor_policy_connected_machine_role" { count = var.enable_arc_servers ? 1 : 0 source = "../dtos-devops-templates/infrastructure/modules/rbac-assignment" - scope = azurerm_resource_group.arc_enabled_servers[0].id + scope = data.azurerm_resource_group.arc_enabled_servers[0].id role_definition_name = "Azure Connected Machine Resource Administrator" principal_id = module.arc_monitor_policy_identity[0].principal_id } -# Grants the identity permission to create DCR associations and configure workspaces +# Grants the identity permission to configure Log Analytics workspaces module "arc_monitor_policy_log_analytics_role" { count = var.enable_arc_servers ? 1 : 0 source = "../dtos-devops-templates/infrastructure/modules/rbac-assignment" - scope = azurerm_resource_group.arc_enabled_servers[0].id + scope = data.azurerm_resource_group.arc_enabled_servers[0].id role_definition_name = "Log Analytics Contributor" principal_id = module.arc_monitor_policy_identity[0].principal_id } +# Grants the identity permission to write DCR associations on Arc machine resources +module "arc_monitor_policy_monitoring_contributor_role" { + count = var.enable_arc_servers ? 1 : 0 + source = "../dtos-devops-templates/infrastructure/modules/rbac-assignment" + + scope = data.azurerm_resource_group.arc_enabled_servers[0].id + role_definition_name = "Monitoring Contributor" + principal_id = module.arc_monitor_policy_identity[0].principal_id +} + # --------------------------------------------------------------------------- # Policy Assignments for Arc-enabled servers # AMA install: "Configure Windows Arc-enabled machines to run Azure Monitor Agent" @@ -98,8 +108,8 @@ resource "azurerm_resource_group_policy_assignment" "ama_install" { count = var.enable_arc_servers ? 1 : 0 name = "ama-install-arc-${var.env_config}" - resource_group_id = azurerm_resource_group.arc_enabled_servers[0].id - policy_definition_id = "/providers/Microsoft.Authorization/policyDefinitions/4efbd9d8-6bc6-45f6-9be2-7fe9dd5d89ff" + resource_group_id = data.azurerm_resource_group.arc_enabled_servers[0].id + policy_definition_id = "/providers/Microsoft.Authorization/policyDefinitions/94f686d6-9a24-4e19-91f1-de937dc171a4" location = var.region identity { @@ -112,8 +122,8 @@ resource "azurerm_resource_group_policy_assignment" "dcr_association" { count = var.enable_arc_servers ? 1 : 0 name = "dcr-assoc-arc-${var.env_config}" - resource_group_id = azurerm_resource_group.arc_enabled_servers[0].id - policy_definition_id = "/providers/Microsoft.Authorization/policyDefinitions/eab1f514-22e3-42e3-9a1f-e1dc9199355c" + resource_group_id = data.azurerm_resource_group.arc_enabled_servers[0].id + policy_definition_id = "/providers/Microsoft.Authorization/policyDefinitions/c24c537f-2516-4c2f-aac5-2cd26baa3d26" location = var.region identity { @@ -126,3 +136,24 @@ resource "azurerm_resource_group_policy_assignment" "dcr_association" { resourceType = { value = "Microsoft.Insights/dataCollectionRules" } }) } + +# Remediation tasks — re-evaluate compliance and deploy AMA/DCR association on +# any non-compliant Arc machines each time Terraform applies. Combined with the +# DeployIfNotExists effect, this makes the monitoring setup self-healing. +resource "azurerm_resource_group_policy_remediation" "ama_install" { + count = var.enable_arc_servers ? 1 : 0 + + name = "remediation-ama-install-${var.env_config}" + resource_group_id = data.azurerm_resource_group.arc_enabled_servers[0].id + policy_assignment_id = azurerm_resource_group_policy_assignment.ama_install[0].id + resource_discovery_mode = "ReEvaluateCompliance" +} + +resource "azurerm_resource_group_policy_remediation" "dcr_association" { + count = var.enable_arc_servers ? 1 : 0 + + name = "remediation-dcr-assoc-${var.env_config}" + resource_group_id = data.azurerm_resource_group.arc_enabled_servers[0].id + policy_assignment_id = azurerm_resource_group_policy_assignment.dcr_association[0].id + resource_discovery_mode = "ReEvaluateCompliance" +} diff --git a/infrastructure/modules/arc-infra/outputs.tf b/infrastructure/modules/arc-infra/outputs.tf index 439c2569..e0f7b0c7 100644 --- a/infrastructure/modules/arc-infra/outputs.tf +++ b/infrastructure/modules/arc-infra/outputs.tf @@ -1,11 +1,11 @@ output "arc_enabled_servers_resource_group_name" { description = "Name of the Arc-enabled servers resource group" - value = var.enable_arc_servers ? azurerm_resource_group.arc_enabled_servers[0].name : null + value = var.enable_arc_servers ? data.azurerm_resource_group.arc_enabled_servers[0].name : null } output "arc_enabled_servers_resource_group_id" { description = "ID of the Arc-enabled servers resource group" - value = var.enable_arc_servers ? azurerm_resource_group.arc_enabled_servers[0].id : null + value = var.enable_arc_servers ? data.azurerm_resource_group.arc_enabled_servers[0].id : null } output "arc_onboarding_spn_client_id" { diff --git a/infrastructure/modules/arc-infra/relay.tf b/infrastructure/modules/arc-infra/relay.tf index a98e51f9..69aee96d 100644 --- a/infrastructure/modules/arc-infra/relay.tf +++ b/infrastructure/modules/arc-infra/relay.tf @@ -15,7 +15,7 @@ locals { data "azurerm_resources" "arc_machines" { count = var.enable_arc_servers ? 1 : 0 - resource_group_name = azurerm_resource_group.arc_enabled_servers[0].name + resource_group_name = data.azurerm_resource_group.arc_enabled_servers[0].name type = "Microsoft.HybridCompute/machines" } diff --git a/infrastructure/terraform/resource_group_init/core.bicep b/infrastructure/terraform/resource_group_init/core.bicep index a2566858..098822c6 100644 --- a/infrastructure/terraform/resource_group_init/core.bicep +++ b/infrastructure/terraform/resource_group_init/core.bicep @@ -8,6 +8,19 @@ param miName string param userGroupPrincipalID string @minLength(1) param userGroupName string +@minLength(1) +param appShortName string +@minLength(1) +param envConfig string +@minLength(1) +param region string + +// Arc-enabled servers RG must exist before Terraform runs so the arc-infra module +// can reference it as a data source rather than managing its lifecycle. +resource arcEnabledServersRG 'Microsoft.Resources/resourceGroups@2024-11-01' = { + name: 'rg-${appShortName}-${envConfig}-uks-arc-enabled-servers' + location: region +} // See: https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles var roleID = { @@ -20,7 +33,7 @@ var roleID = { AzureConnectedMachineOnboarding: 'b64e21ea-ac4e-4cdf-9dc9-5b892992bee7' AzureConnectedMachineResourceAdministrator: 'cd570a14-e51a-42ad-bac8-bafd67325302' logAnalyticsContributor: '92aaf0da-9dab-42b6-94a3-d43ce8d16293' - resourcePolicyContributor: '36243c78-bf99-498c-9df9-ad4ef54afa4c' + resourcePolicyContributor: '36243c78-bf99-498c-9df9-86d9f8d28608' virtualMachineAdministratorLogin: '1c0163c0-47e6-4577-8991-ea5c82e286e4' } diff --git a/infrastructure/terraform/resource_group_init/main.bicep b/infrastructure/terraform/resource_group_init/main.bicep index ec56a3b7..3fade281 100644 --- a/infrastructure/terraform/resource_group_init/main.bicep +++ b/infrastructure/terraform/resource_group_init/main.bicep @@ -77,7 +77,7 @@ module managedIdentiyGHtoADO 'managedIdentity.bicep' = { // Let the GHtoADO managed identity access a subscription resource readerAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().subscriptionId, envConfig, 'reader') + name: guid(subscription().subscriptionId, miGHtoADOname, 'reader') properties: { roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.reader) principalId: managedIdentiyGHtoADO.outputs.miPrincipalID @@ -131,7 +131,7 @@ module storageAccountPrivateEndpoint 'privateEndpoint.bicep' = { // Let the managed identity manage monitoring resources (Application Insights, Log Analytics) resource monitoringContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().subscriptionId, envConfig, 'monitoringContributor') + name: guid(subscription().subscriptionId, miADOtoAZname, 'monitoringContributor') properties: { roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.monitoringContributor) principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID @@ -142,7 +142,7 @@ resource monitoringContributorAssignment 'Microsoft.Authorization/roleAssignment // Let the managed identity configure vnet peering and DNS records resource networkContributorAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(subscription().subscriptionId, envConfig, 'networkContributor') + name: guid(subscription().subscriptionId, miADOtoAZname, 'networkContributor') properties: { roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleID.networkContributor) principalId: managedIdentiyADOtoAZ.outputs.miPrincipalID diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/variables.tf index 786de019..db894346 100644 --- a/infrastructure/terraform/variables.tf +++ b/infrastructure/terraform/variables.tf @@ -33,9 +33,9 @@ variable "enable_gateway_test_vm" { } variable "vnet_address_space" { - description = "Address space for the gateway test VM VNet (e.g., 10.130.0.0/16). Required when enable_gateway_test_vm is true." + description = "Address space for the gateway test VM VNet. This VNet is isolated and never peered — the default works for all environments." type = string - default = null + default = "10.131.0.0/16" } variable "bastion_sku" { diff --git a/scripts/bash/resource_group_init.sh b/scripts/bash/resource_group_init.sh index 4f798e08..22a10d93 100755 --- a/scripts/bash/resource_group_init.sh +++ b/scripts/bash/resource_group_init.sh @@ -48,4 +48,5 @@ echo "Deploy to core subscription $ARM_SUBSCRIPTION_ID..." az deployment sub create --location "$REGION" --template-file infrastructure/terraform/resource_group_init/core.bicep \ --subscription "$ARM_SUBSCRIPTION_ID" \ --parameters miName="$miName" miPrincipalId="$miPrincipalID" \ - userGroupPrincipalID="$userGroupPrincipalID" userGroupName="$userGroupName" --confirm-with-what-if + userGroupPrincipalID="$userGroupPrincipalID" userGroupName="$userGroupName" \ + appShortName="$APP_SHORT_NAME" envConfig="$ENV_CONFIG" region="$REGION" --confirm-with-what-if