fleet-connectivity-status.yml

# AZLOCAL-PIPELINE-ID: fleet-connectivity-status
# Step.04 - Fleet Connectivity Status Monitoring Pipeline
# This pipeline surfaces network-connectivity issues across every Azure Local
# cluster the service connection can read - four scopes in one pipeline:
#
#   1. Cluster Connectivity   - is the cluster reachable from Azure (properties.connectivityStatus)?
#   2. Arc Agent (per node)   - is each physical node's Arc-enabled-server agent Connected?
#   3. Physical NIC issues    - Disconnected adapters with a real (non-APIPA) IP
#   4. Azure Resource Bridge  - is each cluster's ARB appliance Running (and how long offline if not)?
#
# USE CASES:
# - Daily/scheduled fleet-wide network-connectivity audit
# - Find clusters where Arc agent or ARB has dropped off and updates / management calls will fail
# - Catch Disconnected physical NICs that still have a real IP (likely cable / driver / port issues -
#   Disconnected adapters with no IP or APIPA self-assignment are intentionally excluded as noise)
# - Triage hand-off into ITSM (wired identically to Step.9 / Step.10 - opt-in via parameter)
#
# REPORTS GENERATED:
# - Pipeline run summary (uploaded via ##vso[task.uploadsummary]): one section per
#   scope (cluster, ARB, Arc agent, NIC) with portal hyperlinks and emoji severity icons.
#   NIC section lists only issues (Disconnected with non-APIPA IP); the other three
#   scopes show the full universe with healthy + failing rows.
# - JUnit XML: published to the Test Results tab. One <testcase time="0"> per failing resource,
#   grouped under "Cluster Connectivity" / "Arc Agent Connectivity" /
#   "Physical NIC Status" / "Azure Resource Bridge" testsuites.
# - CSV exports per scope: fleet-cluster-connectivity.csv,
#   fleet-arc-status-summary.csv, fleet-arc-non-connected-machines.csv,
#   fleet-physical-nics.csv (issues only), fleet-physical-nic-all.csv (full
#   unfiltered NIC inventory - all NICs, Physical+Virtual, all statuses),
#   fleet-physical-nic-stats.csv (NIC histogram by NicType+NicStatus),
#   fleet-arb-status.csv
# - JSON copies (same shape, machine-readable)
#
# DATA SOURCE:
# All four sections execute Azure Resource Graph queries directly via the
# `az graph query` CLI (the `resource-graph` extension is installed below). No
# per-cluster Az SDK or REST shell-outs - the entire job is ARG-only and runs
# in seconds even on multi-hundred-cluster fleets.
#
# AUTHENTICATION:
# Uses Workload Identity Federation (OIDC) - recommended for secretless authentication.
#
# v0.8.5 thin-YAML: the inline inlineScript body (5 ARG queries via
# Get-AzLocalFleetConnectivityStatus + per-scope severity classification +
# ~200-line inline JUnit StringBuilder + markdown summary render + 12 output
# variables) is now the Export-AzLocalFleetConnectivityStatusReport Public
# cmdlet. This yml is condensed to a few lines per step; the full workload
# (and its Pester tests) live in the module.

# Pipeline display name carries the same Step.N - prefix as the filename so the
# ADO pipelines list (sorts by `name:` field) lists pipelines in execution order.
name: Step.04 - Fleet Connectivity Status

# BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
# Edits inside this block are preserved across Update-AzLocalPipelineExample
# module upgrades. Add or modify `schedules:` blocks below.
trigger: none  # Manual or scheduled only

schedules:
  # Daily at 05:30 UTC - offset 30 min ahead of Step.9 (06:00) and Step.10 (07:00)
  # so the three fleet-scope read pipelines stagger their ARG calls.
  - cron: '30 5 * * *'
    displayName: 'Daily Fleet Connectivity Check'
    branches:
      include:
        - main
    always: true
# END-AZLOCAL-CUSTOMIZE:schedule-triggers

parameters:
  - name: subscriptionFilter
    displayName: 'Optional Azure subscription ID(s) to constrain the ARG query (comma-separated). Leave empty to scan every subscription the federated identity can read.'
    type: string
    default: ''

  - name: moduleVersion
    displayName: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
    type: string
    default: ''

  # --- ITSM Connector (ServiceNow auto-raise on connectivity failures) ---
  # Set raiseItsmTicket=true to open ServiceNow incidents from each Critical /
  # Warning connectivity failure published by this pipeline. Default is false
  # so existing schedules stay byte-identical until you opt in. The connector
  # reads the JUnit file this pipeline produces (fleet-connectivity-status.xml)
  # and the trigger matrix in ./.itsm/azurelocal-itsm.yml. Dedupe granularity is
  # one ticket per (cluster, failing resource) pair.
  - name: raiseItsmTicket
    displayName: 'Open ITSM tickets (ServiceNow) for connectivity failures'
    type: boolean
    default: false

  - name: itsmConfigPath
    displayName: 'Path to ITSM matrix config (YAML or JSON)'
    type: string
    default: './.itsm/azurelocal-itsm.yml'

  - name: itsmDryRun
    displayName: 'ITSM: build payloads + run read-only dedupe but do NOT create tickets'
    type: boolean
    default: false

  - name: itsmForceCreate
    displayName: 'ITSM: bypass dedupe and always create new tickets (use with caution)'
    type: boolean
    default: false

variables:
  # Module version this YAML was generated against. The install step compares this to
  # the version actually installed and to the latest on PSGallery, and emits a warning
  # log if the YAML appears stale - prompting you to refresh via
  # Copy-AzLocalPipelineExample -Update. See Automation-Pipeline-Examples/README.md section 5.
  GENERATED_AGAINST_MODULE_VERSION: '0.8.73'
  # Resolution order for the module version pin (leave all unset to install the latest,
  # which is the default "fix-forward" behaviour): queue-time parameter > pipeline variable
  # 'REQUIRED_MODULE_VERSION' overridden at queue time > empty (latest).
  REQUIRED_MODULE_VERSION: '${{ parameters.moduleVersion }}'
  reportsPath: '$(Build.ArtifactStagingDirectory)/reports'
  # ITSM connector. When raiseItsmTicket=true the Raise ITSM task below
  # references $(ITSM_SN_INSTANCE_URL), $(ITSM_SN_CLIENT_ID),
  # $(ITSM_SN_CLIENT_SECRET). The simplest binding is an ADO variable group
  # named 'AzureLocal-ITSM-Secrets'. ADO does not allow mixing mapping-form
  # variables with '- group:' entries in the same block, so to enable the
  # group reference convert this entire 'variables:' block to list form (see
  # Step.10 for the conversion recipe). When raiseItsmTicket=false the Raise
  # ITSM task is skipped and the secrets are not consulted.

stages:
- stage: FleetConnectivity
  displayName: 'Collect Fleet Connectivity Status'
  jobs:
  - job: CollectConnectivity
    displayName: 'Collect and Report Fleet Connectivity'
    pool:
      vmImage: 'windows-latest'

    steps:
    - checkout: self
      displayName: 'Checkout repository'

    - task: PowerShell@2
      displayName: 'Install AzLocal.UpdateManagement from PSGallery'
      # v0.8.5 thin-YAML: drift detection + banner + output variables are
      # all produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
      name: moduleVersion
      env:
        REQUIRED_MODULE_VERSION: $(REQUIRED_MODULE_VERSION)
        GENERATED_AGAINST_MODULE_VERSION: $(GENERATED_AGAINST_MODULE_VERSION)
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          $ErrorActionPreference = 'Stop'
          $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
          if ($env:REQUIRED_MODULE_VERSION) {
              $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
              Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
          } else {
              Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
          }
          Install-Module @installArgs
          Import-Module AzLocal.UpdateManagement -Force
          Add-AzLocalPipelineVersionBanner `
              -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
              -PinnedVersion $env:REQUIRED_MODULE_VERSION

    - task: AzureCLI@2
      displayName: 'Collect Fleet Connectivity Data'
      # v0.8.5 thin-YAML: the inline inlineScript body (5 ARG queries via
      # Get-AzLocalFleetConnectivityStatus + per-scope severity classification +
      # ~200-line inline JUnit StringBuilder + markdown summary render + 12
      # output variables, named uppercase CLUSTER_TOTAL etc.) has been
      # condensed into the Public cmdlet
      # Export-AzLocalFleetConnectivityStatusReport. The cmdlet writes
      # $(reportsPath)/{7-csv,7-json,fleet-connectivity-status.xml}, emits the
      # markdown summary via task.uploadsummary (using the shared
      # New-AzLocalFleetConnectivityStatusSummary renderer), and sets the 12
      # lowercase output variables on the `collect` task.
      name: collect
      env:
        INPUT_SUBSCRIPTION_FILTER: ${{ parameters.subscriptionFilter }}
        INSTALLED_MODULE_VERSION: $(moduleVersion.installed_module_version)
        REPORTS_PATH: $(reportsPath)
      inputs:
        # Replace with your service connection name
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          $ErrorActionPreference = 'Stop'
          az extension add --name resource-graph --yes
          Import-Module AzLocal.UpdateManagement -Force
          $params = @{
              OutputDirectory        = $env:REPORTS_PATH
              InstalledModuleVersion = $env:INSTALLED_MODULE_VERSION
          }
          if ($env:INPUT_SUBSCRIPTION_FILTER) { $params['SubscriptionFilter'] = $env:INPUT_SUBSCRIPTION_FILTER }
          Export-AzLocalFleetConnectivityStatusReport @params

    - pwsh: |
        $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
        Write-Host "##vso[task.setvariable variable=artifactStamp;isOutput=true]$stamp"
        Write-Host "Artifact timestamp: $stamp"
      name: stamp
      displayName: 'Compute Artifact Timestamp'
      condition: always()

    - task: PublishBuildArtifacts@1
      displayName: 'Publish Fleet Connectivity Reports'
      condition: always()
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.4-fleet-connectivity-status-report_$(stamp.artifactStamp)'
        publishLocation: 'Container'

    # Publish JUnit test results AFTER the cmdlet (which writes the markdown
    # summary first). Mirrors the GitHub Actions twin
    # (dorny/test-reporter@v3 there; PublishTestResults@2 here).
    - task: PublishTestResults@2
      displayName: 'Publish Fleet Connectivity JUnit Diagnostics'
      condition: always()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/fleet-connectivity-status.xml'
        testRunTitle: '[JUnit Debug] Fleet Connectivity Status - $(Build.BuildNumber)'
        mergeTestResults: false
        failTaskOnFailedTests: false  # Do not fail pipeline on connectivity failures

    # ----------------------------------------------------------------------
    # ITSM Connector (ServiceNow auto-raise on connectivity failures)
    # Opt-in (gated on parameters.raiseItsmTicket == true). Mirrors the
    # Step.9 / Step.10 wiring - reads the JUnit file produced above, uses
    # the same azurelocal-itsm.yml trigger matrix, same dedupe key.
    # ----------------------------------------------------------------------
    - task: PowerShell@2
      displayName: 'Install powershell-yaml (ITSM config parser)'
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      inputs:
        targetType: 'inline'
        pwsh: true
        script: |
          if (-not (Get-Module -ListAvailable -Name powershell-yaml)) {
              Install-Module powershell-yaml -Scope CurrentUser -Force -AllowClobber
          }

    - task: AzureCLI@2
      displayName: 'Raise ITSM tickets'
      name: itsm
      condition: and(succeeded(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      env:
        # BEGIN-AZLOCAL-CUSTOMIZE:itsm-secrets
        # Bind your ITSM connector secrets from the variable group above.
        # Defaults assume the ServiceNow OAuth client_credentials naming used
        # by azurelocal-itsm.yml (env://ITSM_SN_*). If your secret names differ,
        # update them here - this block is preserved by
        # Update-AzLocalPipelineExample across module upgrades.
        ITSM_SN_INSTANCE_URL: $(ITSM_SN_INSTANCE_URL)
        ITSM_SN_CLIENT_ID: $(ITSM_SN_CLIENT_ID)
        ITSM_SN_CLIENT_SECRET: $(ITSM_SN_CLIENT_SECRET)
        # END-AZLOCAL-CUSTOMIZE:itsm-secrets
      inputs:
        azureSubscription: 'AzureLocal-ServiceConnection'  # Update with your service connection name
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          $ErrorActionPreference = 'Stop'
          Import-Module AzLocal.UpdateManagement -Force

          $configPath = "${{ parameters.itsmConfigPath }}"
          $dryRun     = [bool]::Parse("${{ parameters.itsmDryRun }}")
          $force      = [bool]::Parse("${{ parameters.itsmForceCreate }}")

          if (-not (Test-Path $configPath)) {
              Write-Host "##vso[task.logissue type=warning]ITSM config not found at '$configPath' - skipping ticket creation."
              exit 0
          }

          $cfg = Get-AzLocalItsmConfig -Path $configPath

          $junitInput = "$(reportsPath)/fleet-connectivity-status.xml"
          if (-not (Test-Path $junitInput)) {
              Write-Host "##vso[task.logissue type=warning]No fleet-connectivity-status.xml found at '$junitInput' - skipping ticket creation."
              exit 0
          }

          $params = @{
              InputArtifactPath = $junitInput
              Config            = $cfg
              RunMetadata       = @{
                  Platform = 'azure-devops'
                  RunId    = $env:BUILD_BUILDID
                  RunUrl   = "$env:SYSTEM_COLLECTIONURI$env:SYSTEM_TEAMPROJECT/_build/results?buildId=$env:BUILD_BUILDID"
                  Branch   = $env:BUILD_SOURCEBRANCH
              }
              DryRun            = $dryRun
              ForceCreate       = $force
              ExportPath        = "$(reportsPath)/itsm-results.csv"
              ExportJUnitPath   = "$(reportsPath)/itsm-results.xml"
          }

          $results = New-AzLocalIncident @params
          $results | Format-Table ClusterName, Action, TicketId, Severity -AutoSize

    - task: PublishBuildArtifacts@1
      displayName: 'Publish ITSM Artefacts'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      continueOnError: true
      inputs:
        PathtoPublish: '$(reportsPath)'
        ArtifactName: 'azlocal-step.4-fleet-connectivity-status-itsm-results_$(stamp.artifactStamp)'
        publishLocation: 'Container'

    - task: PublishTestResults@2
      displayName: 'Publish ITSM Test Results'
      condition: and(always(), eq('${{ parameters.raiseItsmTicket }}', 'true'))
      continueOnError: true
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '$(reportsPath)/itsm-results.xml'
        testRunTitle: 'ITSM Tickets (Step.4) - $(Build.BuildNumber)'
        mergeTestResults: false
        failTaskOnFailedTests: false