monitor-updates.yml

# AZLOCAL-PIPELINE-ID: monitor-updates
# Step.08 - Monitor In-Flight Updates (Azure DevOps)
# -----------------------------------------------------------
# Reports clusters whose latest update run is currently in flight, with the
# CURRENT STEP each cluster is on and the ELAPSED DURATION of the run. Built
# for operators who want a 30-minute-cadence "what is happening right now"
# view, distinct from the daily Step.9 fleet-update-status snapshot which
# answers "is each cluster up-to-date?".
#
# This pipeline is REPORT-ONLY and always succeeds. Long-running runs are
# surfaced via:
#   - The pipeline summary tab (markdown uploaded via task.uploadsummary).
#   - update-monitor.csv (full per-cluster rows for spreadsheet triage).
#   - update-monitor.xml (JUnit, one <testcase time="0"> per in-flight cluster;
#     <failure> when the elapsed duration exceeds the threshold so the Tests
#     tab shows stuck runs in red).
#
# Data source: Get-AzLocalUpdateRuns -Latest (one row per cluster).
#
# AUTHENTICATION:
# Uses Workload Identity Federation (OIDC) - recommended for secretless authentication.
# See: https://learn.microsoft.com/en-us/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-using-workload-identity-federation

# BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
# Edits inside this block are preserved across Update-AzLocalPipelineExample
# module upgrades. Add or modify `schedules:` blocks below.
trigger: none  # Manual or scheduled only

# Default (v0.7.92+): runs 5x/day, every 2 hours across the typical overnight
# maintenance window (20:00, 22:00, 00:00, 02:00, 04:00 UTC). Azure DevOps
# cron is ALWAYS UTC - adjust the hour list below for your local maintenance
# window. Comment the `schedules:` block out entirely to disable.
#
# ADO does not support sub-hour cron in YAML `schedules:` - the lowest
# cadence is hourly. For sub-hour cadence, use a pipeline trigger from an
# external scheduler (e.g. an Azure Function) that queues this pipeline via REST.
schedules:
  - cron: '0 0,2,4,20,22 * * *'
    displayName: 'In-flight update monitor (5x/day, 20-04 UTC every 2h)'
    branches:
      include:
        - main
    always: true
# END-AZLOCAL-CUSTOMIZE:schedule-triggers

parameters:
  - name: scope
    displayName: 'Scope of clusters to monitor'
    type: string
    default: 'all'
    values:
      - 'all'
      - 'by-update-ring'

  - name: updateRing
    # accepts single ring, 'Prod;Ring2' list, or '***' wildcard (three stars - deliberate).
    displayName: "UpdateRing tag value (only used when scope=by-update-ring). Single, 'Prod;Ring2', or '***'."
    type: string
    default: ''

  - name: longRunningStepHours
    displayName: 'PRIMARY stuck-step signal. Flag in-flight runs whose CURRENT STEP has been running longer than this many hours.'
    type: string
    default: '2'

  - name: longRunningThresholdHours
    displayName: 'Belt-and-braces overall-elapsed flag (default 24h).'
    type: string
    default: '24'

  - name: recentFailureWindowHours
    displayName: 'Surface FAILED runs whose End time falls within the last N hours (0 = disable).'
    type: string
    default: '24'

  - name: criticalElapsedDays
    displayName: 'CRITICAL tier for overall-elapsed (default 3 days). Runs older than this get a :rotating_light: chip; older than 2x get a :skull: chip - strongest visual signal that human intervention is needed.'
    type: string
    default: '3'

  - name: moduleVersion
    displayName: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
    type: string
    default: ''

variables:
  GENERATED_AGAINST_MODULE_VERSION: '0.8.77'
  REQUIRED_MODULE_VERSION: '${{ parameters.moduleVersion }}'
  reportsPath: '$(Build.ArtifactStagingDirectory)/reports'

stages:
- stage: MonitorInFlight
  displayName: 'Monitor In-Flight Update Runs'
  jobs:
  - job: Monitor
    displayName: 'Snapshot in-flight update runs'
    pool:
      vmImage: 'windows-latest'

    steps:
    - checkout: self
      displayName: 'Checkout repository'

    - task: PowerShell@2
      name: moduleVersionStep
      displayName: 'Install AzLocal.UpdateManagement from PSGallery'
      # v0.8.5 thin-YAML: drift detection + banner + step outputs are all
      # produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
      env:
        REQUIRED_MODULE_VERSION: $(REQUIRED_MODULE_VERSION)
        GENERATED_AGAINST_MODULE_VERSION: $(GENERATED_AGAINST_MODULE_VERSION)
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          $ErrorActionPreference = 'Stop'
          $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
          if ($env:REQUIRED_MODULE_VERSION) {
              $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
              Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
          } else {
              Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
          }
          Install-Module @installArgs
          Import-Module AzLocal.UpdateManagement -Force
          Add-AzLocalPipelineVersionBanner `
              -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
              -PinnedVersion $env:REQUIRED_MODULE_VERSION

    - task: AzureCLI@2
      name: snapshot
      displayName: 'Snapshot in-flight update runs'
      # v0.8.5 thin-YAML: the inline run block (cluster scope query via
      # Get-AzLocalUpdateRuns -Latest -PassThru, severity classification,
      # per-cluster CSV, JUnit XML, the six pipeline variables, and the
      # markdown step summary with status badge + in-flight + unresolved-
      # failed tables) has been condensed into the Public cmdlet
      # Export-AzLocalUpdateRunMonitorReport. The cmdlet writes
      # $(reportsPath)/update-monitor.csv + .xml, uploads the markdown
      # summary via ##vso[task.uploadsummary], and sets the six pipeline
      # variables (in_flight, long_running, long_running_step,
      # step_errored, recent_failures, unresolved_failures).
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          az extension add --name resource-graph --yes
          Import-Module AzLocal.UpdateManagement -Force
          $params = @{
              Scope                  = "${{ parameters.scope }}"
              OutputDirectory        = "$(reportsPath)"
              InstalledModuleVersion = "$(moduleVersionStep.installed_module_version)"
          }
          if ("${{ parameters.updateRing }}") { $params['UpdateRing'] = "${{ parameters.updateRing }}" }
          [int]$parsed = 0
          if ([int]::TryParse("${{ parameters.longRunningStepHours }}",      [ref]$parsed) -and $parsed -gt 0) { $params['LongRunningStepHours']      = $parsed }
          if ([int]::TryParse("${{ parameters.longRunningThresholdHours }}", [ref]$parsed) -and $parsed -gt 0) { $params['LongRunningThresholdHours'] = $parsed }
          if ([int]::TryParse("${{ parameters.recentFailureWindowHours }}",  [ref]$parsed) -and $parsed -ge 0) { $params['RecentFailureWindowHours']  = $parsed }
          if ([int]::TryParse("${{ parameters.criticalElapsedDays }}",       [ref]$parsed) -and $parsed -gt 0) { $params['CriticalElapsedDays']       = $parsed }
          Export-AzLocalUpdateRunMonitorReport @params

    # compute a UTC timestamp variable so every downloadable artifact name is unique per run.
    - pwsh: |
        $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
        Write-Host "##vso[task.setvariable variable=artifactStamp;isOutput=true]$stamp"
        Write-Host "Artifact timestamp: $stamp"
      displayName: 'Compute Artifact Timestamp'
      condition: always()
      name: stamp

    - task: PublishBuildArtifacts@1
      displayName: 'Publish in-flight monitor reports'
      condition: always()
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.8-update-monitor_$(stamp.artifactStamp)'
        publishLocation: 'Container'

    - task: PublishTestResults@2
      displayName: 'Publish in-flight monitor JUnit'
      condition: always()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/update-monitor.xml'
        testRunTitle: 'In-Flight Update Monitor'
        mergeTestResults: false
        failTaskOnFailedTests: false