monitor-updates.yml

# AZLOCAL-PIPELINE-ID: monitor-updates
# Update: 4 - Monitor In-Flight Updates (Azure DevOps)
# -----------------------------------------------------------
# Reports clusters whose latest update run is currently in flight, with the
# CURRENT STEP each cluster is on and the ELAPSED DURATION of the run. Built
# for operators who want a 30-minute-cadence "what is happening right now"
# view, distinct from the daily Step.9 fleet-update-status snapshot which
# answers "is each cluster up-to-date?".
#
# This pipeline is REPORT-ONLY and always succeeds. Long-running runs are
# surfaced via:
#   - The pipeline summary tab (markdown uploaded via task.uploadsummary).
#   - update-monitor.csv (full per-cluster rows for spreadsheet triage).
#   - update-monitor.xml (JUnit, one <testcase time="0"> per in-flight cluster;
#     <failure> when the elapsed duration exceeds the threshold so the Tests
#     tab shows stuck runs in red).
#
# Data source: Get-AzLocalUpdateRuns -Latest (one row per cluster).
#
# AUTHENTICATION:
# Uses Workload Identity Federation (OIDC) - recommended for secretless authentication.
# See: https://learn.microsoft.com/en-us/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-using-workload-identity-federation

# BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
# Edits inside this block are preserved across Update-AzLocalPipelineExample
# module upgrades. Add or modify `schedules:` blocks below.
trigger: none  # Manual or scheduled only

# Default (v0.7.92+): runs 5x/day, every 2 hours across the typical overnight
# maintenance window (20:00, 22:00, 00:00, 02:00, 04:00 UTC). Azure DevOps
# cron is ALWAYS UTC - adjust the hour list below for your local maintenance
# window. Comment the `schedules:` block out entirely to disable.
#
# ADO does not support sub-hour cron in YAML `schedules:` - the lowest
# cadence is hourly. For sub-hour cadence, use a pipeline trigger from an
# external scheduler (e.g. an Azure Function) that queues this pipeline via REST.
schedules:
  - cron: '0 0,2,4,20,22 * * *'
    displayName: 'In-flight update monitor (5x/day, 20-04 UTC every 2h)'
    branches:
      include:
        - main
    always: true
# END-AZLOCAL-CUSTOMIZE:schedule-triggers

parameters:
  - name: scope
    displayName: 'Scope of clusters to monitor'
    type: string
    default: 'all'
    values:
      - 'all'
      - 'by-update-ring'

  - name: updateRing
    # accepts single ring, 'Prod;Ring2' list, or '***' wildcard (three stars - deliberate).
    displayName: "UpdateRing tag value (only used when scope=by-update-ring). Single, 'Prod;Ring2', or '***'."
    type: string
    default: ''

  - name: longRunningStepHours
    displayName: 'PRIMARY stuck-step signal. Flag in-flight runs whose CURRENT STEP has been running longer than this many hours.'
    type: string
    default: '2'

  - name: longRunningThresholdHours
    displayName: 'Belt-and-braces overall-elapsed flag (default 24h).'
    type: string
    default: '24'

  - name: recentFailureWindowHours
    displayName: 'Surface FAILED runs whose End time falls within the last N hours (0 = disable).'
    type: string
    default: '24'

  - name: criticalElapsedDays
    displayName: 'CRITICAL tier for overall-elapsed (default 3 days). Runs older than this get a :rotating_light: chip; older than 2x get a :skull: chip - strongest visual signal that human intervention is needed.'
    type: string
    default: '3'

  - name: moduleVersion
    displayName: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
    type: string
    default: ''

  # --- ITSM Connector (ServiceNow auto-raise on in-flight monitor failures) ---
  # Set raiseItsmTicket=true to open ServiceNow incidents (v0.8.87) from each
  # stuck-step / failed / attempt-without-run cluster surfaced in
  # update-monitor.xml. Default false so existing schedules stay byte-identical
  # until you opt in. The connector reads the JUnit file this pipeline already
  # produces (update-monitor.xml) and the trigger matrix in
  # ./.itsm/azurelocal-itsm.yml (defaults raise AttemptWithoutRun / StepError /
  # Failed; LongRunningStep / LongRunningOverall are opt-in).
  - name: raiseItsmTicket
    displayName: 'Open ITSM tickets (ServiceNow) for in-flight monitor failures'
    type: boolean
    default: false

  - name: itsmConfigPath
    displayName: 'Path to ITSM matrix config (YAML or JSON)'
    type: string
    default: './.itsm/azurelocal-itsm.yml'

  - name: itsmDryRun
    displayName: 'ITSM: build payloads + run read-only dedupe but do NOT create tickets'
    type: boolean
    default: false

  - name: itsmForceCreate
    displayName: 'ITSM: bypass dedupe and always create new tickets (use with caution)'
    type: boolean
    default: false

variables:
  GENERATED_AGAINST_MODULE_VERSION: '0.8.87'
  REQUIRED_MODULE_VERSION: '${{ parameters.moduleVersion }}'
  reportsPath: '$(Build.ArtifactStagingDirectory)/reports'
  # ITSM connector. When raiseItsmTicket=true the Raise ITSM task below
  # references $(ITSM_SN_INSTANCE_URL), $(ITSM_SN_CLIENT_ID),
  # $(ITSM_SN_CLIENT_SECRET). The simplest binding is an ADO variable group
  # named 'AzureLocal-ITSM-Secrets'. ADO does not allow mixing mapping-form
  # variables with '- group:' entries in the same block, so to enable the
  # group reference convert this entire 'variables:' block to list form:
  #   variables:
  #     - name: GENERATED_AGAINST_MODULE_VERSION
  #       value: '0.8.87'
  #     - name: REQUIRED_MODULE_VERSION
  #       value: '${{ parameters.moduleVersion }}'
  #     - name: reportsPath
  #       value: '$(Build.ArtifactStagingDirectory)/reports'
  #     - group: AzureLocal-ITSM-Secrets
  # When raiseItsmTicket=false the Raise ITSM task is skipped and the secrets
  # are not consulted, so the conversion is only required for opt-in users.

stages:
- stage: MonitorInFlight
  displayName: 'Monitor In-Flight Update Runs'
  jobs:
  - job: Monitor
    displayName: 'Snapshot in-flight update runs'
    pool:
      vmImage: 'windows-latest'

    steps:
    - checkout: self
      displayName: 'Checkout repository'

    - task: PowerShell@2
      name: moduleVersionStep
      displayName: 'Install AzLocal.UpdateManagement from PSGallery'
      # v0.8.5 thin-YAML: drift detection + banner + step outputs are all
      # produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
      env:
        REQUIRED_MODULE_VERSION: $(REQUIRED_MODULE_VERSION)
        GENERATED_AGAINST_MODULE_VERSION: $(GENERATED_AGAINST_MODULE_VERSION)
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          $ErrorActionPreference = 'Stop'
          $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
          if ($env:REQUIRED_MODULE_VERSION) {
              $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
              Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
          } else {
              Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
          }
          Install-Module @installArgs
          Import-Module AzLocal.UpdateManagement -Force
          Add-AzLocalPipelineVersionBanner `
              -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
              -PinnedVersion $env:REQUIRED_MODULE_VERSION

    - task: AzureCLI@2
      name: snapshot
      displayName: 'Snapshot in-flight update runs'
      # v0.8.5 thin-YAML: the inline run block (cluster scope query via
      # Get-AzLocalUpdateRuns -Latest -PassThru, severity classification,
      # per-cluster CSV, JUnit XML, the six pipeline variables, and the
      # markdown step summary with status badge + in-flight + unresolved-
      # failed tables) has been condensed into the Public cmdlet
      # Export-AzLocalUpdateRunMonitorReport. The cmdlet writes
      # $(reportsPath)/update-monitor.csv + .xml, uploads the markdown
      # summary via ##vso[task.uploadsummary], and sets the seven pipeline
      # variables (in_flight, long_running, long_running_step,
      # step_errored, recent_failures, unresolved_failures,
      # attempts_without_run).
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          az extension add --name resource-graph --yes
          Import-Module AzLocal.UpdateManagement -Force
          $params = @{
              Scope                  = "${{ parameters.scope }}"
              OutputDirectory        = "$(reportsPath)"
              InstalledModuleVersion = "$(moduleVersionStep.installed_module_version)"
          }
          if ("${{ parameters.updateRing }}") { $params['UpdateRing'] = "${{ parameters.updateRing }}" }
          [int]$parsed = 0
          if ([int]::TryParse("${{ parameters.longRunningStepHours }}",      [ref]$parsed) -and $parsed -gt 0) { $params['LongRunningStepHours']      = $parsed }
          if ([int]::TryParse("${{ parameters.longRunningThresholdHours }}", [ref]$parsed) -and $parsed -gt 0) { $params['LongRunningThresholdHours'] = $parsed }
          if ([int]::TryParse("${{ parameters.recentFailureWindowHours }}",  [ref]$parsed) -and $parsed -ge 0) { $params['RecentFailureWindowHours']  = $parsed }
          if ([int]::TryParse("${{ parameters.criticalElapsedDays }}",       [ref]$parsed) -and $parsed -gt 0) { $params['CriticalElapsedDays']       = $parsed }
          Export-AzLocalUpdateRunMonitorReport @params

    # compute a UTC timestamp variable so every downloadable artifact name is unique per run.
    - pwsh: |
        $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
        Write-Host "##vso[task.setvariable variable=artifactStamp;isOutput=true]$stamp"
        Write-Host "Artifact timestamp: $stamp"
      displayName: 'Compute Artifact Timestamp'
      condition: always()
      name: stamp

    - task: PublishBuildArtifacts@1
      displayName: 'Publish in-flight monitor reports'
      condition: always()
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.8-update-monitor_$(stamp.artifactStamp)'
        publishLocation: 'Container'

    - task: PublishTestResults@2
      displayName: 'Publish in-flight monitor JUnit'
      condition: always()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/update-monitor.xml'
        testRunTitle: 'In-Flight Update Monitor'
        mergeTestResults: false
        failTaskOnFailedTests: false

    # ----------------------------------------------------------------------
    # ITSM Connector (ServiceNow auto-raise on in-flight monitor failures)
    #
    # Fully opt-in (gated on parameters.raiseItsmTicket == true). Runs AFTER
    # the JUnit file is published, so the ITSM task is strictly additive -
    # failures here never affect the monitor result (which is always green;
    # this is a report-only pipeline). Reads the same update-monitor.xml that
    # powers the summary. The JUnit emitter writes per-testcase <properties>
    # (ClusterName / ClusterResourceId / UpdateName / Status / CurrentStep /
    # ClusterPortalUrl / UpdateRunPortalUrl) so New-AzLocalIncident can compute
    # the SHA256 dedupe key (one incident per stuck/failed/attempt-gap cluster
    # until it clears) and the Mustache body template can deep-link into the
    # Azure portal.
    # ----------------------------------------------------------------------
    - task: PowerShell@2
      displayName: 'Install powershell-yaml (ITSM config parser)'
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          if (-not (Get-Module -ListAvailable -Name powershell-yaml)) {
              Install-Module powershell-yaml -Scope CurrentUser -Force -AllowClobber
          }

    - task: AzureCLI@2
      displayName: 'Raise ITSM tickets'
      name: itsm
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      env:
        # BEGIN-AZLOCAL-CUSTOMIZE:itsm-secrets
        # Bind your ITSM connector secrets from the variable group above.
        # Defaults assume the ServiceNow OAuth client_credentials naming used
        # by azurelocal-itsm.yml (env://ITSM_SN_*). If your secret names differ,
        # update them here - this block is preserved by
        # Update-AzLocalPipelineExample across module upgrades.
        ITSM_SN_INSTANCE_URL: $(ITSM_SN_INSTANCE_URL)
        ITSM_SN_CLIENT_ID: $(ITSM_SN_CLIENT_ID)
        ITSM_SN_CLIENT_SECRET: $(ITSM_SN_CLIENT_SECRET)
        # END-AZLOCAL-CUSTOMIZE:itsm-secrets
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          Import-Module AzLocal.UpdateManagement -Force

          $configPath = "${{ parameters.itsmConfigPath }}"
          $dryRun     = [System.Convert]::ToBoolean("${{ parameters.itsmDryRun }}")
          $force      = [System.Convert]::ToBoolean("${{ parameters.itsmForceCreate }}")

          if (-not (Test-Path $configPath)) {
              Write-Host "##vso[task.logissue type=warning]ITSM config not found at '$configPath' - skipping ticket creation."
              exit 0
          }

          $cfg = Get-AzLocalItsmConfig -Path $configPath

          $junitInput = "$(reportsPath)/update-monitor.xml"
          if (-not (Test-Path $junitInput)) {
              Write-Host "##vso[task.logissue type=warning]No update-monitor.xml found at '$junitInput' - skipping ticket creation."
              exit 0
          }

          $params = @{
              InputArtifactPath = $junitInput
              Config            = $cfg
              RunMetadata       = @{
                  Platform = 'azure-devops'
                  RunId    = $env:BUILD_BUILDID
                  RunUrl   = "$($env:SYSTEM_COLLECTIONURI)$($env:SYSTEM_TEAMPROJECT)/_build/results?buildId=$($env:BUILD_BUILDID)"
                  Branch   = $env:BUILD_SOURCEBRANCH
              }
              DryRun            = $dryRun
              ForceCreate       = $force
              ExportPath        = "$(reportsPath)/itsm-results.csv"
              ExportJUnitPath   = "$(reportsPath)/itsm-results.xml"
          }

          $results = New-AzLocalIncident @params
          $results | Format-Table ClusterName, Action, TicketId, Severity -AutoSize

    - task: PublishBuildArtifacts@1
      displayName: 'Publish ITSM Artefacts'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.8-update-monitor-itsm-results_$(stamp.artifactStamp)'
        publishLocation: 'Container'
      continueOnError: true

    - task: PublishTestResults@2
      displayName: 'Publish ITSM Test Results'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/itsm-results.xml'
        testRunTitle: 'ITSM Tickets (Update: 4 Monitor) - $(Build.BuildNumber)'
        failTaskOnFailedTests: false