fleet-health-status.yml

# AZLOCAL-PIPELINE-ID: fleet-health-status
# Step.10 - Fleet Health Status Monitoring Pipeline
# This pipeline surfaces 24-hour system health-check failures across every Azure Local
# cluster the service connection can read - including clusters that are already
# "up to date" with no available updates. The 24-hour health checks continue to run
# on the cluster even when no update is in flight, so this pipeline is the dedicated
# place to triage fleet-wide health issues that exist OUTSIDE the update workflow.
#
# USE CASES:
# - Daily / weekly fleet-wide Critical & Warning health audit
# - Executive dashboard: "which failure reason hits the most clusters?"
# - Compliance tracking for cluster health, independent of update activity
# - Triage hand-off into ITSM: the JUnit XML and CSV exports are ready-made
#   incident payloads
#
# REPORTS GENERATED:
# - Pipeline run summary (uploaded via ##vso[task.uploadsummary]): pivoted by
#   FailureReason (top failures by cluster impact), with a per-cluster "Detailed
#   Results" table further down.
# - JUnit XML (diagnostic mirror): published to the Test Results tab.
#   One <testcase time="0"> per (cluster, failing health check), grouped under
#   "[JUnit Debug] Critical Health Failures" /
#   "[JUnit Debug] Warning Health Failures" testsuites for two-level drill-down.
# - CSV (detail):  fleet-health-detail.csv  - one row per (cluster, failing check)
# - CSV (summary): fleet-health-summary.csv - one row per (FailureReason, Severity)
# - JSON: same shape, machine-readable for downstream automation
#
# AUTHENTICATION:
# Uses Workload Identity Federation (OIDC) - recommended for secretless authentication.

# BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
# Edits inside this block are preserved across Update-AzLocalPipelineExample
# module upgrades. Add or modify `schedules:` blocks below.
trigger: none  # Manual or scheduled only

schedules:
  # Run daily at 7 AM UTC (offset from fleet-update-status which runs at 06:00)
  - cron: '0 7 * * *'
    displayName: 'Daily Fleet Health Check'
    branches:
      include:
        - main
    always: true
# END-AZLOCAL-CUSTOMIZE:schedule-triggers

parameters:
  - name: scope
    displayName: 'Scope of clusters to check'
    type: string
    default: 'all'
    values:
      - 'all'
      - 'by-update-ring'

  - name: updateRing
    # accepts single ring, 'Prod;Ring2' list, or '***' wildcard (three stars - deliberate).
    displayName: "UpdateRing tag value (only used when scope=by-update-ring). Single, 'Prod;Ring2', or '***'."
    type: string
    default: ''

  - name: severity
    displayName: 'Severity filter at Resource Graph (default All = Critical + Warning; Informational always excluded)'
    type: string
    default: 'All'
    values:
      - 'All'
      - 'Critical'
      - 'Warning'

  - name: moduleVersion
    displayName: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
    type: string
    default: ''

  # --- ITSM Connector (ServiceNow auto-raise on fleet-health failures) ---
  # Set raiseItsmTicket=true to open ServiceNow incidents from each Critical /
  # Warning health-check failure published by this pipeline. Default is false
  # so existing schedules stay byte-identical until you opt in. The connector
  # reads the JUnit file this pipeline already produces (fleet-health-status.xml) and
  # the trigger matrix in ./.itsm/azurelocal-itsm.yml. Dedupe granularity is
  # one ticket per (cluster, failing health check) pair - FailureReason is fed
  # into the UpdateName slot of the SHA256 dedupe key.
  - name: raiseItsmTicket
    displayName: 'Open ITSM tickets (ServiceNow) for fleet-health failures'
    type: boolean
    default: false

  - name: itsmConfigPath
    displayName: 'Path to ITSM matrix config (YAML or JSON)'
    type: string
    default: './.itsm/azurelocal-itsm.yml'

  - name: itsmDryRun
    displayName: 'ITSM: build payloads + run read-only dedupe but do NOT create tickets'
    type: boolean
    default: false

  - name: itsmForceCreate
    displayName: 'ITSM: bypass dedupe and always create new tickets (use with caution)'
    type: boolean
    default: false

variables:
  # Module version this YAML was generated against. The install step compares this to
  # the version actually installed and to the latest on PSGallery, and emits a warning
  # log if the YAML appears stale - prompting you to refresh via
  # Copy-AzLocalPipelineExample -Update. See Automation-Pipeline-Examples/README.md section 5.
  GENERATED_AGAINST_MODULE_VERSION: '0.8.73'
  # Resolution order for the module version pin (leave all unset to install the latest,
  # which is the default "fix-forward" behaviour): queue-time parameter > pipeline variable
  # 'REQUIRED_MODULE_VERSION' overridden at queue time > empty (latest).
  REQUIRED_MODULE_VERSION: '${{ parameters.moduleVersion }}'
  reportsPath: '$(Build.ArtifactStagingDirectory)/reports'
  # ITSM connector. When raiseItsmTicket=true the Raise ITSM
  # task below references $(ITSM_SN_INSTANCE_URL), $(ITSM_SN_CLIENT_ID),
  # $(ITSM_SN_CLIENT_SECRET). The simplest binding is an ADO variable group
  # named 'AzureLocal-ITSM-Secrets'. ADO does not allow mixing mapping-form
  # variables with '- group:' entries in the same block, so to enable the
  # group reference convert this entire 'variables:' block to list form:
  #   variables:
  #     - name: GENERATED_AGAINST_MODULE_VERSION
  #       value: '0.7.78'
  #     - name: REQUIRED_MODULE_VERSION
  #       value: '${{ parameters.moduleVersion }}'
  #     - name: reportsPath
  #       value: '$(Build.ArtifactStagingDirectory)/reports'
  #     - group: AzureLocal-ITSM-Secrets
  # When raiseItsmTicket=false the Raise ITSM task is skipped and the secrets
  # are not consulted, so the conversion is only required for opt-in users.

stages:
- stage: FleetHealth
  displayName: 'Collect Fleet Health Status'
  jobs:
  - job: CollectHealth
    displayName: 'Collect and Report Fleet Health'
    pool:
      vmImage: 'windows-latest'

    steps:
    - checkout: self
      displayName: 'Checkout repository'

    - task: PowerShell@2
      displayName: 'Install AzLocal.UpdateManagement from PSGallery'
      # v0.8.5 thin-YAML: drift detection + banner + step outputs are all
      # produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
      name: moduleVersion
      env:
        REQUIRED_MODULE_VERSION: $(REQUIRED_MODULE_VERSION)
        GENERATED_AGAINST_MODULE_VERSION: $(GENERATED_AGAINST_MODULE_VERSION)
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          $ErrorActionPreference = 'Stop'
          $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
          if ($env:REQUIRED_MODULE_VERSION) {
              $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
              Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
          } else {
              Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
          }
          Install-Module @installArgs
          Import-Module AzLocal.UpdateManagement -Force
          Add-AzLocalPipelineVersionBanner `
              -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
              -PinnedVersion $env:REQUIRED_MODULE_VERSION


    - task: AzureCLI@2
      displayName: 'Collect Fleet Health Status'
      # v0.8.5 thin-YAML: the inline run block (Get-AzLocalFleetHealthFailures
      # Detail view + in-process Group-Object summary roll-up +
      # Get-AzLocalFleetHealthOverview + 2-suite JUnit XML emission +
      # 4-section markdown step summary written via ##vso[task.uploadsummary] +
      # 8 step outputs) has been condensed into the Public cmdlet
      # Export-AzLocalFleetHealthStatusReport. The cmdlet writes
      # $(reportsPath)/*.{json,csv,xml} for downstream Publish + ITSM steps.
      name: collect
      env:
        INPUT_SCOPE: ${{ parameters.scope }}
        INPUT_UPDATE_RING: ${{ parameters.updateRing }}
        INPUT_SEVERITY: ${{ parameters.severity }}
        INSTALLED_MODULE_VERSION: $(moduleVersion.installed_module_version)
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          $ErrorActionPreference = 'Stop'
          az extension add --name resource-graph --yes
          Import-Module AzLocal.UpdateManagement -Force
          $params = @{
              Scope                  = if ($env:INPUT_SCOPE) { $env:INPUT_SCOPE } else { 'all' }
              Severity               = if ($env:INPUT_SEVERITY) { $env:INPUT_SEVERITY } else { 'All' }
              OutputDirectory        = "$(reportsPath)"
              InstalledModuleVersion = $env:INSTALLED_MODULE_VERSION
          }
          if ($env:INPUT_UPDATE_RING) { $params['UpdateRing'] = $env:INPUT_UPDATE_RING }
          Export-AzLocalFleetHealthStatusReport @params


    # compute a UTC timestamp variable so every downloadable artifact name is unique per run.
    - pwsh: |
        $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
        Write-Host "##vso[task.setvariable variable=artifactStamp;isOutput=true]$stamp"
        Write-Host "Artifact timestamp: $stamp"
      displayName: 'Compute Artifact Timestamp'
      condition: always()
      name: stamp

    # Publish artifacts first (lightweight), then summary, then JUnit. Summary
    # is uploaded via ##vso[task.uploadsummary] in the next step, which makes it
    # the first thing visible on the run extensions tab.
    - task: PublishBuildArtifacts@1
      displayName: 'Publish Fleet Health Reports'
      condition: always()
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.10-fleet-health-status-report_$(stamp.artifactStamp)'
        publishLocation: 'Container'

    # Publish JUnit test results LAST so the Test Results tab is populated but the
    # markdown summary above is the first thing visible on the run extensions view.
    - task: PublishTestResults@2
      displayName: 'Publish Fleet Health JUnit Diagnostics'
      condition: always()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/fleet-health-status.xml'
        testRunTitle: 'Fleet Health Status [JUnit Debug] - $(Build.BuildNumber)'
        mergeTestResults: false
        failTaskOnFailedTests: false  # Don't fail pipeline on health failures

    # ----------------------------------------------------------------------
    # ITSM Connector (ServiceNow auto-raise on fleet-health failures)
    #
    # Fully opt-in (gated on parameters.raiseItsmTicket == true). Runs AFTER
    # PublishTestResults so the fleet-health-status surfaces its own results
    # first; ITSM is strictly additive - failures here never affect the
    # pipeline exit code.
    #
    # Reads the same fleet-health-status.xml that powers the run summary.
    # The JUnit emitter writes per-testcase <properties> (ClusterName /
    # ClusterResourceId / UpdateName=FailureReason / Status=Severity /
    # FailureReason / Severity / ClusterPortalUrl / TargetResourceName /
    # TargetResourceType) so New-AzLocalIncident can compute the SHA256
    # dedupe key (one ticket per cluster + failing check) and the Mustache
    # body template can deep-link straight into the cluster blade.
    # ----------------------------------------------------------------------
    - task: PowerShell@2
      displayName: 'Install powershell-yaml (ITSM config parser)'
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          if (-not (Get-Module -ListAvailable -Name powershell-yaml)) {
              Install-Module powershell-yaml -Scope CurrentUser -Force -AllowClobber
          }

    - task: AzureCLI@2
      displayName: 'Raise ITSM tickets'
      name: itsm
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      env:
        # BEGIN-AZLOCAL-CUSTOMIZE:itsm-secrets
        # Bind your ITSM connector secrets from the variable group above.
        # Defaults assume the ServiceNow OAuth client_credentials naming used
        # by azurelocal-itsm.yml (env://ITSM_SN_*). If your secret names differ,
        # update them here - this block is preserved by
        # Update-AzLocalPipelineExample across module upgrades.
        ITSM_SN_INSTANCE_URL: $(ITSM_SN_INSTANCE_URL)
        ITSM_SN_CLIENT_ID: $(ITSM_SN_CLIENT_ID)
        ITSM_SN_CLIENT_SECRET: $(ITSM_SN_CLIENT_SECRET)
        # END-AZLOCAL-CUSTOMIZE:itsm-secrets
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          Import-Module AzLocal.UpdateManagement -Force

          $configPath = "${{ parameters.itsmConfigPath }}"
          $dryRun     = [System.Convert]::ToBoolean("${{ parameters.itsmDryRun }}")
          $force      = [System.Convert]::ToBoolean("${{ parameters.itsmForceCreate }}")

          if (-not (Test-Path $configPath)) {
              Write-Host "##vso[task.logissue type=warning]ITSM config not found at '$configPath' - skipping ticket creation."
              exit 0
          }

          $cfg = Get-AzLocalItsmConfig -Path $configPath

          $junitInput = "$(reportsPath)/fleet-health-status.xml"
          if (-not (Test-Path $junitInput)) {
              Write-Host "##vso[task.logissue type=warning]No fleet-health-status.xml found at '$junitInput' - skipping ticket creation."
              exit 0
          }

          $params = @{
              InputArtifactPath = $junitInput
              Config            = $cfg
              RunMetadata       = @{
                  Platform = 'azure-devops'
                  RunId    = $env:BUILD_BUILDID
                  RunUrl   = "$($env:SYSTEM_COLLECTIONURI)$($env:SYSTEM_TEAMPROJECT)/_build/results?buildId=$($env:BUILD_BUILDID)"
                  Branch   = $env:BUILD_SOURCEBRANCH
              }
              DryRun            = $dryRun
              ForceCreate       = $force
              ExportPath        = "$(reportsPath)/itsm-results.csv"
              ExportJUnitPath   = "$(reportsPath)/itsm-results.xml"
          }

          $results = New-AzLocalIncident @params
          $results | Format-Table ClusterName, Action, TicketId, Severity -AutoSize

    - task: PublishBuildArtifacts@1
      displayName: 'Publish ITSM Artefacts'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.10-fleet-health-status-itsm-results_$(stamp.artifactStamp)'
        publishLocation: 'Container'
      continueOnError: true

    - task: PublishTestResults@2
      displayName: 'Publish ITSM Test Results'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/itsm-results.xml'
        testRunTitle: 'ITSM Tickets (Step.10) - $(Build.BuildNumber)'
        failTaskOnFailedTests: false