fleet-update-status.yml

# AZLOCAL-PIPELINE-ID: fleet-update-status
# Step.09 - Fleet Update Status Monitoring Pipeline
# This pipeline monitors update status across all Azure Local clusters and generates reports
#
# USE CASES:
# - Daily/weekly health checks on cluster update status
# - Executive dashboards showing fleet-wide update adoption
# - Alerting when clusters have update failures
# - Compliance tracking for update deployments
#
# REPORTS GENERATED:
# - JUnit XML: Shows each cluster as a test case (passed/failed/skipped) in Test tab
#   (testcase bodies now include `Start Time:` and `End Time:` lines per cluster).
# - CSV: Detailed export for spreadsheet analysis (includes `EndTime` column sourced
#   from properties.progress.endTimeUtc, with fallback to properties.lastUpdatedTime - blank for InProgress runs).
# - HTML report (Recent Update Run History section): includes `End Time` column for the
#   aggregated multi-attempt row.
# - JSON: Machine-readable format for custom integrations
#
# AUTHENTICATION:
# Uses Workload Identity Federation (OIDC) - recommended for secretless authentication
# See: https://learn.microsoft.com/en-us/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-using-workload-identity-federation

# BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
# Edits inside this block are preserved across Update-AzLocalPipelineExample
# module upgrades. Add or modify `schedules:` blocks below.
trigger: none  # Manual or scheduled only

schedules:
  # Run daily at 6 AM UTC
  - cron: '0 6 * * *'
    displayName: 'Daily Fleet Status Check'
    branches:
      include:
        - main
    always: true  # Run even if no code changes
# END-AZLOCAL-CUSTOMIZE:schedule-triggers

parameters:
  - name: scope
    displayName: 'Scope of clusters to check'
    type: string
    default: 'all'
    values:
      - 'all'
      - 'by-update-ring'
  
  - name: updateRing
    # accepts single ring, 'Prod;Ring2' list, or '***' wildcard (three stars - deliberate).
    displayName: "UpdateRing tag value (only used when scope=by-update-ring). Single, 'Prod;Ring2', or '***'."
    type: string
    default: ''
  
  - name: includeUpdateRuns
    displayName: 'Include recent update run history in report'
    type: boolean
    default: true

  - name: moduleVersion
    displayName: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
    type: string
    default: ''

  # --- ITSM Connector (ServiceNow auto-raise on fleet-update-status) ---
  # Set raiseItsmTicket=true to open ServiceNow incidents from each unresolved
  # Failed update run reported by the 'Update Run History and Error Details'
  # testsuite (and any failed AzureLocalFleetUpdateStatus row). Default is false
  # so existing schedules stay byte-identical until you opt in. The connector
  # reads the JUnit file Step.7 already produces (readiness-status.xml) and
  # the trigger matrix in ./.itsm/azurelocal-itsm.yml.
  - name: raiseItsmTicket
    displayName: 'Open ITSM tickets (ServiceNow) for fleet-update-status failures'
    type: boolean
    default: false

  - name: itsmConfigPath
    displayName: 'Path to ITSM matrix config (YAML or JSON)'
    type: string
    default: './.itsm/azurelocal-itsm.yml'

  - name: itsmDryRun
    displayName: 'ITSM: build payloads + run read-only dedupe but do NOT create tickets'
    type: boolean
    default: false

  - name: itsmForceCreate
    displayName: 'ITSM: bypass dedupe and always create new tickets (use with caution)'
    type: boolean
    default: false

variables:
  # Module version this YAML was generated against. The install step compares this to
  # the version actually installed and to the latest on PSGallery, and emits a warning
  # log if the YAML appears stale - prompting you to refresh via
  # Copy-AzLocalPipelineExample -Update. See Automation-Pipeline-Examples/README.md section 5.
  GENERATED_AGAINST_MODULE_VERSION: '0.8.83'
  # Resolution order for the module version pin (leave all unset to install the latest,
  # which is the default "fix-forward" behaviour): queue-time parameter > pipeline variable
  # 'REQUIRED_MODULE_VERSION' overridden at queue time > empty (latest).
  REQUIRED_MODULE_VERSION: '${{ parameters.moduleVersion }}'
  reportsPath: '$(Build.ArtifactStagingDirectory)/reports'
  # ITSM connector. When raiseItsmTicket=true the Raise ITSM
  # task below references $(ITSM_SN_INSTANCE_URL), $(ITSM_SN_CLIENT_ID),
  # $(ITSM_SN_CLIENT_SECRET). The simplest binding is an ADO variable group
  # named 'AzureLocal-ITSM-Secrets'. ADO does not allow mixing mapping-form
  # variables with '- group:' entries in the same block, so to enable the
  # group reference convert this entire 'variables:' block to list form:
  #   variables:
  #     - name: GENERATED_AGAINST_MODULE_VERSION
  #       value: '0.7.77'
  #     - name: REQUIRED_MODULE_VERSION
  #       value: '${{ parameters.moduleVersion }}'
  #     - name: reportsPath
  #       value: '$(Build.ArtifactStagingDirectory)/reports'
  #     - group: AzureLocal-ITSM-Secrets
  # When raiseItsmTicket=false the Raise ITSM task is skipped and the secrets
  # are not consulted, so the conversion is only required for opt-in users.

stages:
- stage: FleetStatus
  displayName: 'Collect Fleet Update Status'
  jobs:
  - job: CollectStatus
    displayName: 'Collect and Report Fleet Status'
    pool:
      vmImage: 'windows-latest'
    
    steps:
    - checkout: self
      displayName: 'Checkout repository'
    
    - task: PowerShell@2
      displayName: 'Install AzLocal.UpdateManagement from PSGallery'
      # v0.8.5 thin-YAML: drift detection + banner + step outputs are all
      # produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
      name: moduleVersion
      env:
        REQUIRED_MODULE_VERSION: $(REQUIRED_MODULE_VERSION)
        GENERATED_AGAINST_MODULE_VERSION: $(GENERATED_AGAINST_MODULE_VERSION)
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          $ErrorActionPreference = 'Stop'
          $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
          if ($env:REQUIRED_MODULE_VERSION) {
              $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
              Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
          } else {
              Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
          }
          Install-Module @installArgs
          Import-Module AzLocal.UpdateManagement -Force
          Add-AzLocalPipelineVersionBanner `
              -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
              -PinnedVersion $env:REQUIRED_MODULE_VERSION
    
    # Use Azure CLI task with Workload Identity Federation (OIDC)
    - task: AzureCLI@2
      displayName: 'Collect Fleet Update Status'
      # v0.8.5 thin-YAML: the inline run block (cluster inventory, readiness
      # cascade, version distribution with Microsoft-manifest support window,
      # 3-suite JUnit XML, supplementary CSV/JSON exports, run history collection,
      # 22 step outputs, and the full markdown step summary uploaded via
      # ##vso[task.uploadsummary]) has been condensed into the Public cmdlet
      # Export-AzLocalFleetUpdateStatusReport. The cmdlet writes
      # $(reportsPath)/*.{json,csv,xml} for downstream Publish + ITSM steps.
      name: collect
      env:
        INPUT_SCOPE: ${{ parameters.scope }}
        INPUT_UPDATE_RING: ${{ parameters.updateRing }}
        INPUT_INCLUDE_UPDATE_RUNS: ${{ parameters.includeUpdateRuns }}
        INSTALLED_MODULE_VERSION: $(moduleVersion.installed_module_version)
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          $ErrorActionPreference = 'Stop'
          Import-Module AzLocal.UpdateManagement -Force
          $params = @{
              Scope                  = if ($env:INPUT_SCOPE) { $env:INPUT_SCOPE } else { 'all' }
              IncludeUpdateRuns      = ($env:INPUT_INCLUDE_UPDATE_RUNS -ne 'false')
              InstalledModuleVersion = $env:INSTALLED_MODULE_VERSION
          }
          if ($env:INPUT_UPDATE_RING) { $params['UpdateRing'] = $env:INPUT_UPDATE_RING }
          Export-AzLocalFleetUpdateStatusReport @params

    # compute a UTC timestamp variable so every downloadable artifact name is unique per run.
    - pwsh: |
        $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
        Write-Host "##vso[task.setvariable variable=artifactStamp;isOutput=true]$stamp"
        Write-Host "Artifact timestamp: $stamp"
      displayName: 'Compute Artifact Timestamp'
      condition: always()
      name: stamp

    # Publish all reports as artifacts FIRST (lightweight) so the summary
    # task below runs before the JUnit Publish Test Results task, ensuring
    # the markdown summary uploaded via ##vso[task.uploadsummary] appears
    # at the top of the pipeline run extensions view.
    - task: PublishBuildArtifacts@1
      displayName: 'Publish Fleet Status Reports'
      condition: always()
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.9-fleet-update-status-report_$(stamp.artifactStamp)'
        publishLocation: 'Container'
    
    # Publish JUnit test results LAST so the Test Results tab is populated
    # but the markdown summary above is the first thing visible on the run
    # extensions view. failTaskOnFailedTests is left $false because cluster
    # health/update failures are surfaced via the summary; the pipeline
    # itself only fails on unrecoverable workflow errors.
    - task: PublishTestResults@2
      displayName: 'Publish Fleet Status as Test Results'
      condition: always()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/readiness-status.xml'
        testRunTitle: 'Fleet Update Status - $(Build.BuildNumber)'
        mergeTestResults: false
        failTaskOnFailedTests: false  # Don't fail pipeline on test failures

    # ----------------------------------------------------------------------
    # ITSM Connector (ServiceNow auto-raise on fleet failures)
    #
    # Fully opt-in (gated on parameters.raiseItsmTicket == true). Runs AFTER
    # PublishTestResults so the fleet-update-status surfaces its own results
    # first; ITSM is strictly additive - failures here never affect the
    # pipeline exit code.
    #
    # Reads the same readiness-status.xml that powers the run summary. The
    # JUnit emitter writes per-testcase <properties> (ClusterName /
    # ClusterResourceId / UpdateName / Status / ErrorCategory / CurrentStep /
    # Duration / UpdateRunPortalUrl / ClusterPortalUrl) so New-AzLocalIncident
    # can compute the SHA256 dedupe key and the Mustache body template can
    # deep-link straight into the SingleInstanceHistoryDetails ReactView for
    # the failed update run.
    # ----------------------------------------------------------------------
    - task: PowerShell@2
      displayName: 'Install powershell-yaml (ITSM config parser)'
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          if (-not (Get-Module -ListAvailable -Name powershell-yaml)) {
              Install-Module powershell-yaml -Scope CurrentUser -Force -AllowClobber
          }

    - task: AzureCLI@2
      displayName: 'Raise ITSM tickets'
      name: itsm
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      env:
        # BEGIN-AZLOCAL-CUSTOMIZE:itsm-secrets
        # Bind your ITSM connector secrets from the variable group above.
        # Defaults assume the ServiceNow OAuth client_credentials naming used
        # by azurelocal-itsm.yml (env://ITSM_SN_*). If your secret names differ,
        # update them here - this block is preserved by
        # Update-AzLocalPipelineExample across module upgrades.
        ITSM_SN_INSTANCE_URL: $(ITSM_SN_INSTANCE_URL)
        ITSM_SN_CLIENT_ID: $(ITSM_SN_CLIENT_ID)
        ITSM_SN_CLIENT_SECRET: $(ITSM_SN_CLIENT_SECRET)
        # END-AZLOCAL-CUSTOMIZE:itsm-secrets
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          Import-Module AzLocal.UpdateManagement -Force

          $configPath = "${{ parameters.itsmConfigPath }}"
          $dryRun     = [System.Convert]::ToBoolean("${{ parameters.itsmDryRun }}")
          $force      = [System.Convert]::ToBoolean("${{ parameters.itsmForceCreate }}")

          if (-not (Test-Path $configPath)) {
              Write-Host "##vso[task.logissue type=warning]ITSM config not found at '$configPath' - skipping ticket creation."
              exit 0
          }

          $cfg = Get-AzLocalItsmConfig -Path $configPath

          $junitInput = "$(reportsPath)/readiness-status.xml"
          if (-not (Test-Path $junitInput)) {
              Write-Host "##vso[task.logissue type=warning]No readiness-status.xml found at '$junitInput' - skipping ticket creation."
              exit 0
          }

          $params = @{
              InputArtifactPath = $junitInput
              Config            = $cfg
              RunMetadata       = @{
                  Platform = 'azure-devops'
                  RunId    = $env:BUILD_BUILDID
                  RunUrl   = "$($env:SYSTEM_COLLECTIONURI)$($env:SYSTEM_TEAMPROJECT)/_build/results?buildId=$($env:BUILD_BUILDID)"
                  Branch   = $env:BUILD_SOURCEBRANCH
              }
              DryRun            = $dryRun
              ForceCreate       = $force
              ExportPath        = "$(reportsPath)/itsm-results.csv"
              ExportJUnitPath   = "$(reportsPath)/itsm-results.xml"
          }

          $results = New-AzLocalIncident @params
          $results | Format-Table ClusterName, Action, TicketId, Severity -AutoSize

    - task: PublishBuildArtifacts@1
      displayName: 'Publish ITSM Artefacts'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.9-fleet-update-status-itsm-results_$(stamp.artifactStamp)'
        publishLocation: 'Container'
      continueOnError: true

    - task: PublishTestResults@2
      displayName: 'Publish ITSM Test Results'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/itsm-results.xml'
        testRunTitle: 'ITSM Tickets (Step.9) - $(Build.BuildNumber)'
        failTaskOnFailedTests: false