monitor-updates.yml

# AZLOCAL-PIPELINE-ID: monitor-updates
# Monitor In-Flight Updates (operational monitoring snapshot)
# -----------------------------------------------------------
# Reports clusters whose latest update run is currently in flight, with the
# CURRENT STEP each cluster is on and the ELAPSED DURATION of the run. Built
# for operators who want a 30-minute-cadence "what is happening right now"
# view, distinct from the daily Step.9 fleet-update-status snapshot which
# answers "is each cluster up-to-date?".
#
# This workflow is REPORT-ONLY and always succeeds. Long-running runs are
# surfaced via:
#   - The markdown step summary (sorted by elapsed duration descending, with
#     a Threshold column flagging runs that have exceeded the configured
#     long-running threshold).
#   - update-monitor.csv (full per-cluster rows for spreadsheet triage).
#   - update-monitor.xml (JUnit, one <testcase time="0"> per in-flight cluster;
#     <failure> when the elapsed duration exceeds the threshold so the
#     Checks tab shows stuck runs in red).
#
# Data source: Get-AzLocalUpdateRuns -Latest (one row per cluster). No new
# cmdlets required - this is a pure pipeline composition.

# Workflow name carries the same Step.N - prefix as the filename so the GitHub
# Actions sidebar (which sorts workflows alphabetically by this `name:` field)
# lists the pipelines in execution order.
name: Step.08 - Monitor In-Flight Updates

on:
  # BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
  # Content between BEGIN/END markers is preserved by
  # Update-AzLocalPipelineExample across module upgrades.
  #
  # Default (v0.7.92+): runs 5x/day, every 2 hours across the typical overnight
  # maintenance window (20:00, 22:00, 00:00, 02:00, 04:00 UTC). GitHub Actions
  # cron is ALWAYS UTC - adjust the hour list below for your local maintenance
  # window. Comment the `schedule:` block out entirely to disable.
  #
  # Other typical patterns:
  #   '*/30 * * * *'         every 30 min, 24x7 (busiest)
  #   '0 */2 * * *'          every 2 hours, 24x7
  #   '*/30 * * * 0,6'       every 30 min, weekends only
  #   '*/15 22-6 * * 0,6'    every 15 min during the weekend maintenance window
  schedule:
    - cron: '0 0,2,4,20,22 * * *'
  # END-AZLOCAL-CUSTOMIZE:schedule-triggers

  workflow_dispatch:
    inputs:
      scope:
        description: 'Scope of clusters to monitor'
        required: true
        default: 'all'
        type: choice
        options:
          - 'all'
          - 'by-update-ring'
      update_ring:
        # accepts a single ring (Wave1), a semicolon-delimited list (Prod;Ring2),
        # or '***' (three stars - deliberate, not a typo) to match every cluster that HAS the UpdateRing tag set.
        description: "UpdateRing tag value (only used when scope=by-update-ring). Single ring, 'Prod;Ring2', or '***'."
        required: false
        default: ''
      long_running_step_hours:
        description: 'PRIMARY stuck-step signal. Flag in-flight runs whose CURRENT STEP has been running longer than this many hours (default 2). More accurate than overall-run elapsed for large clusters where a legitimate 12-node run can exceed 6-8h while each individual step ticks along normally.'
        required: false
        default: '2'
      long_running_threshold_hours:
        description: 'Belt-and-braces overall-elapsed flag (default 24h). Useful as a backstop for very long-stuck runs; per-step elapsed (long_running_step_hours) is the primary stuck-run signal.'
        required: false
        default: '24'
      recent_failure_window_hours:
        description: 'Surface FAILED runs whose End time falls within the last N hours (default 24). Set to 0 to disable.'
        required: false
        default: '24'
      critical_elapsed_days:
        description: 'CRITICAL tier for overall-elapsed (default 3 days). In-flight runs older than this get a :rotating_light: chip; older than 2x this get a :skull: chip. Strongest visual signal that human intervention is needed - a 19-day-stuck run will no longer be hidden behind a single "step errored" chip.'
        required: false
        default: '3'
      module_version:
        description: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
        required: false
        default: ''

env:
  # Module version this workflow YAML was generated against. The install step compares
  # this to the version actually installed and to the latest on PSGallery, and emits a
  # ::notice annotation if the YAML appears stale - prompting you to refresh via
  # Copy-AzLocalPipelineExample -Update. See Automation-Pipeline-Examples/README.md section 5.
  GENERATED_AGAINST_MODULE_VERSION: '0.8.78'
  # Resolution order for the module version pin (leave all unset to install the latest,
  # which is the default "fix-forward" behaviour): manual workflow_dispatch input >
  # repository variable 'REQUIRED_MODULE_VERSION' > empty (latest).
  REQUIRED_MODULE_VERSION: ${{ github.event.inputs.module_version || vars.REQUIRED_MODULE_VERSION || '' }}
  # v0.8.4 - opt this workflow into Node.js 24 for all JavaScript actions
  # (actions/checkout, actions/download-artifact, actions/upload-artifact,
  # azure/login, dorny/test-reporter, etc). Per GitHub's 2025-09-19 deprecation
  # notice Node 20 is forced off by default on 2026-06-16 and removed from the
  # runner on 2026-09-16. Setting this env var silences the deprecation warnings
  # and exercises Node 24 ahead of the cut-over. To temporarily opt back out,
  # set ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true.
  # https://github.blog/changelog/2025-09-19-deprecation-of-node-20-on-github-actions-runners/
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  monitor:
    name: Monitor in-flight update runs
    runs-on: windows-latest
    permissions:
      id-token: write
      contents: read
      checks: write
    outputs:
      # v0.8.5 thin-YAML: keys align byte-for-byte with the lowercase output
      # names emitted by Export-AzLocalUpdateRunMonitorReport (which uses
      # Set-AzLocalPipelineOutput under the hood). ADO is case-sensitive on
      # task.setvariable lookups; GH is case-insensitive but the lowercase
      # form is the canonical one used by every v0.8.5+ Step.* cmdlet.
      in_flight: ${{ steps.snapshot.outputs.in_flight }}
      long_running: ${{ steps.snapshot.outputs.long_running }}
      long_running_step: ${{ steps.snapshot.outputs.long_running_step }}
      step_errored: ${{ steps.snapshot.outputs.step_errored }}
      recent_failures: ${{ steps.snapshot.outputs.recent_failures }}
      unresolved_failures: ${{ steps.snapshot.outputs.unresolved_failures }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v5

      - name: Azure CLI Login (OIDC)
        uses: azure/login@v3
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ vars.AZURE_TENANT_ID }}
          # AZURE_SUBSCRIPTION_ID is a repository *Variable* (vars.*), not a Secret. It
          # is consumed ONLY here: azure/login@v3 runs `az account set --subscription
          # <id>` after the OIDC token exchange so the runner has a default
          # `az account` context. It is NOT used to scope Azure Resource Graph queries
          # (those run fleet-wide across every subscription the federated identity can
          # read) and is NOT interpolated into Azure portal deep-link URLs (those use
          # the per-row `subscriptionId` returned by ARG).
          # Set it via: gh variable set AZURE_SUBSCRIPTION_ID --body <subId>
          subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}

      - name: Install Azure CLI Resource Graph Extension
        shell: pwsh
        run: az extension add --name resource-graph --yes

      - name: Install AzLocal.UpdateManagement from PSGallery
        # v0.8.5 thin-YAML: drift detection + banner + step outputs are all
        # produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
        shell: pwsh
        id: module-version
        run: |
          $ErrorActionPreference = 'Stop'
          $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
          if ($env:REQUIRED_MODULE_VERSION) {
              $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
              Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
          } else {
              Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
          }
          Install-Module @installArgs
          Import-Module AzLocal.UpdateManagement -Force
          Add-AzLocalPipelineVersionBanner `
              -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
              -PinnedVersion $env:REQUIRED_MODULE_VERSION

      - name: Snapshot in-flight update runs
        # v0.8.5 thin-YAML: the inline run block (cluster scope query
        # via Get-AzLocalUpdateRuns -Latest -PassThru, severity
        # classification, per-cluster CSV, JUnit XML, the six step
        # outputs, and the markdown step summary with status badge +
        # in-flight + unresolved-failed tables) has been condensed into
        # the Public cmdlet Export-AzLocalUpdateRunMonitorReport. The
        # cmdlet writes ./reports/update-monitor.csv + .xml, emits the
        # markdown summary via GITHUB_STEP_SUMMARY, and sets the six
        # step outputs (in_flight, long_running, long_running_step,
        # step_errored, recent_failures, unresolved_failures).
        id: snapshot
        shell: pwsh
        env:
          INPUT_SCOPE: ${{ github.event.inputs.scope }}
          INPUT_UPDATE_RING: ${{ github.event.inputs.update_ring }}
          INPUT_THRESHOLD_HOURS: ${{ github.event.inputs.long_running_threshold_hours }}
          INPUT_STEP_THRESHOLD_HOURS: ${{ github.event.inputs.long_running_step_hours }}
          INPUT_RECENT_FAILURE_WINDOW_HOURS: ${{ github.event.inputs.recent_failure_window_hours }}
          INPUT_CRITICAL_ELAPSED_DAYS: ${{ github.event.inputs.critical_elapsed_days }}
          INSTALLED_MODULE_VERSION: ${{ steps.module-version.outputs.installed_module_version }}
        run: |
          $ErrorActionPreference = 'Stop'
          Import-Module AzLocal.UpdateManagement -Force
          $params = @{
              Scope                  = if ($env:INPUT_SCOPE) { $env:INPUT_SCOPE } else { 'all' }
              OutputDirectory        = './reports'
              InstalledModuleVersion = $env:INSTALLED_MODULE_VERSION
          }
          if ($env:INPUT_UPDATE_RING) { $params['UpdateRing'] = $env:INPUT_UPDATE_RING }
          [int]$parsed = 0
          if ([int]::TryParse([string]$env:INPUT_STEP_THRESHOLD_HOURS, [ref]$parsed) -and $parsed -gt 0)        { $params['LongRunningStepHours']      = $parsed }
          if ([int]::TryParse([string]$env:INPUT_THRESHOLD_HOURS, [ref]$parsed) -and $parsed -gt 0)             { $params['LongRunningThresholdHours'] = $parsed }
          if ([int]::TryParse([string]$env:INPUT_RECENT_FAILURE_WINDOW_HOURS, [ref]$parsed) -and $parsed -ge 0) { $params['RecentFailureWindowHours']  = $parsed }
          if ([int]::TryParse([string]$env:INPUT_CRITICAL_ELAPSED_DAYS, [ref]$parsed) -and $parsed -gt 0)       { $params['CriticalElapsedDays']       = $parsed }
          Export-AzLocalUpdateRunMonitorReport @params

      - name: Compute Artifact Timestamp
        if: always()
        id: artifact-stamp
        shell: pwsh
        run: |
          $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
          "timestamp=$stamp" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
          Write-Host "Artifact timestamp: $stamp"

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: azlocal-step.8-update-monitor_${{ steps.artifact-stamp.outputs.timestamp }}
          path: ./reports/*
          retention-days: 14

      - name: Publish Update Monitor JUnit
        if: always()
        uses: dorny/test-reporter@v3
        with:
          name: 'In-Flight Update Monitor'
          path: ./reports/update-monitor.xml
          reporter: java-junit
          list-suites: failed
          list-tests: failed
        continue-on-error: true