fleet-connectivity-status.yml

# AZLOCAL-PIPELINE-ID: fleet-connectivity-status
# Fleet Connectivity Status Monitoring
# This workflow surfaces network-connectivity issues across every Azure Local
# cluster the service principal can read - four scopes in one pipeline:
#
#   1. Cluster Connectivity   - is the cluster reachable from Azure (properties.connectivityStatus)?
#   2. Arc Agent (per node)   - is each physical node's Arc-enabled-server agent Connected?
#   3. Physical NIC issues    - Disconnected adapters with a real (non-APIPA) IP
#   4. Azure Resource Bridge  - is each cluster's ARB appliance Running (and how long offline if not)?
#
# USE CASES:
# - Daily/scheduled fleet-wide network-connectivity audit
# - Find clusters where Arc agent or ARB has dropped off and updates / management calls will fail
# - Catch Disconnected physical NICs that still have a real IP (likely cable / driver / port issues -
#   Disconnected adapters with no IP or APIPA self-assignment are intentionally excluded as noise)
# - Triage hand-off into ITSM (wired identically to Step.9 / Step.10 - opt-in via input)
#
# REPORTS GENERATED:
# - Markdown step summary: one section per scope (cluster, ARB, Arc agents, NICs)
#   with portal hyperlinks and Critical / Warning emoji icons. NIC section lists
#   only issues (Disconnected with non-APIPA IP); the other three sections show
#   the full universe with healthy + failing rows.
# - JUnit XML: testsuites grouped by Critical (Disconnected / Offline) and Warning
#   (NotConnected / Expired / partial), one testcase per failing resource. dorny
#   test-reporter renders both severities so reviewers can drill suite -> resource.
# - CSV exports per scope: fleet-cluster-connectivity.csv,
#   fleet-arc-status-summary.csv, fleet-arc-non-connected-machines.csv,
#   fleet-physical-nics.csv (issues only), fleet-physical-nic-all.csv (full
#   unfiltered NIC inventory - all NICs, Physical+Virtual, all statuses),
#   fleet-physical-nic-stats.csv (NIC histogram by NicType+NicStatus),
#   fleet-arb-status.csv
# - JSON copies of the above (machine-readable)
#
# DATA SOURCE:
# All four sections execute Azure Resource Graph queries directly via the
# `az graph query` CLI (the `resource-graph` extension is installed below). No
# per-cluster Az SDK or REST shell-outs - the entire job is ARG-only and runs
# in seconds even on multi-hundred-cluster fleets.
#
# AUTHENTICATION:
# Uses OpenID Connect (OIDC) - recommended for secretless authentication
# See: https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure-openid-connect
#
# v0.8.5 thin-YAML: the inline run: | body (data collection +
# severity classification + JUnit XML generation + markdown summary +
# step outputs) is now the Export-AzLocalFleetConnectivityStatusReport
# Public cmdlet. This yml is condensed to a few lines per step; the
# full workload (and its Pester tests) live in the module.

# Workflow name carries the same Step.N - prefix as the filename so the GitHub
# Actions sidebar (which sorts workflows alphabetically by this `name:` field)
# lists the pipelines in execution order.
name: 'Fleet: 02 - Fleet Connectivity Status'

on:
  # BEGIN-AZLOCAL-CUSTOMIZE:schedule-triggers
  # Edits inside this block are preserved across Update-AzLocalPipelineExample
  # module upgrades. Add or modify `schedule:` cron blocks below.
  # Daily at 05:30 UTC - offset 30 min ahead of Step.9 (06:00) and Step.10 (07:00)
  # so the three fleet-scope read pipelines stagger their ARG calls.
  schedule:
    - cron: '30 5 * * *'  # Daily at 05:30 UTC
  # END-AZLOCAL-CUSTOMIZE:schedule-triggers

  # Manual trigger with options
  workflow_dispatch:
    inputs:
      subscription_filter:
        description: 'Optional Azure subscription ID(s) to constrain the ARG query (comma-separated). Leave empty to scan every subscription the federated identity can read.'
        required: false
        default: ''
      module_version:
        description: 'Pin AzLocal.UpdateManagement version (empty = latest from PSGallery). See Automation-Pipeline-Examples/README.md section 5 "Optional configuration".'
        required: false
        default: ''
      # --- ITSM Connector (ServiceNow auto-raise on connectivity failures) ---
      # Set raise_itsm_ticket=true to open ServiceNow incidents from each
      # Critical / Warning connectivity failure published by this pipeline.
      # Default is false so existing schedules stay byte-identical until you
      # opt in. The connector reads the JUnit file this pipeline produces
      # (./reports/fleet-connectivity-status.xml) and the trigger matrix in
      # ./.itsm/azurelocal-itsm.yml. Dedupe granularity is one ticket per
      # (cluster, failing resource) pair.
      raise_itsm_ticket:
        description: 'Open ITSM tickets (ServiceNow) for connectivity failures'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'true'
      itsm_config_path:
        description: 'Path to ITSM matrix config (YAML or JSON)'
        required: false
        default: './.itsm/azurelocal-itsm.yml'
      itsm_dry_run:
        description: 'ITSM: build payloads + run read-only dedupe but do NOT create tickets'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'true'
      itsm_force_create:
        description: 'ITSM: bypass dedupe and always create new tickets (use with caution)'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'true'

env:
  # Module version this workflow YAML was generated against. The install step
  # compares this to the version actually installed and to the latest on
  # PSGallery, and emits a ::notice annotation if the YAML appears stale -
  # prompting you to refresh via Copy-AzLocalPipelineExample -Update. See
  # Automation-Pipeline-Examples/README.md section 5.
  GENERATED_AGAINST_MODULE_VERSION: '0.8.85'
  # Resolution order for the module version pin (leave all unset to install
  # the latest, the default "fix-forward" behaviour): manual workflow_dispatch
  # input > repository variable 'REQUIRED_MODULE_VERSION' > empty (latest).
  REQUIRED_MODULE_VERSION: ${{ github.event.inputs.module_version || vars.REQUIRED_MODULE_VERSION || '' }}
  # v0.8.4 - opt this workflow into Node.js 24 for all JavaScript actions
  # (actions/checkout, actions/download-artifact, actions/upload-artifact,
  # azure/login, dorny/test-reporter, etc). Per GitHub's 2025-09-19 deprecation
  # notice Node 20 is forced off by default on 2026-06-16 and removed from the
  # runner on 2026-09-16. Setting this env var silences the deprecation warnings
  # and exercises Node 24 ahead of the cut-over. To temporarily opt back out,
  # set ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true.
  # https://github.blog/changelog/2025-09-19-deprecation-of-node-20-on-github-actions-runners/
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  fleet-connectivity:
    name: Collect Fleet Connectivity Status
    runs-on: windows-latest
    permissions:
      id-token: write
      contents: read
      checks: write  # Required for test reporter
    outputs:
      # v0.8.5 thin-YAML: keys align byte-for-byte with the lowercase output
      # names emitted by Export-AzLocalFleetConnectivityStatusReport (via
      # Set-AzLocalPipelineOutput). ADO is case-sensitive on task.setvariable
      # lookups; GH is case-insensitive but lowercase is the canonical v0.8.5+
      # convention.
      cluster_total:  ${{ steps.fleet-connectivity.outputs.cluster_total }}
      cluster_fail:   ${{ steps.fleet-connectivity.outputs.cluster_fail }}
      arc_total:      ${{ steps.fleet-connectivity.outputs.arc_total }}
      arc_fail:       ${{ steps.fleet-connectivity.outputs.arc_fail }}
      nic_total:      ${{ steps.fleet-connectivity.outputs.nic_total }}
      nic_fail:       ${{ steps.fleet-connectivity.outputs.nic_fail }}
      nic_all_total:  ${{ steps.fleet-connectivity.outputs.nic_all_total }}
      arb_total:      ${{ steps.fleet-connectivity.outputs.arb_total }}
      arb_fail:       ${{ steps.fleet-connectivity.outputs.arb_fail }}
      total_failures: ${{ steps.fleet-connectivity.outputs.total_failures }}
      critical_count: ${{ steps.fleet-connectivity.outputs.critical_count }}
      warning_count:  ${{ steps.fleet-connectivity.outputs.warning_count }}

    steps:
    - name: Checkout repository
      uses: actions/checkout@v5

    # OIDC Authentication - recommended
    - name: Azure CLI Login (OIDC)
      uses: azure/login@v3
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        tenant-id: ${{ vars.AZURE_TENANT_ID }}
        # AZURE_SUBSCRIPTION_ID is a repository *Variable* (vars.*), not a
        # Secret. It is consumed ONLY here: azure/login@v3 runs `az account
        # set --subscription <id>` after the OIDC token exchange so the
        # runner has a default `az account` context. It is NOT used to scope
        # Azure Resource Graph queries (those run fleet-wide via
        # `--subscriptions` below across every subscription the federated
        # identity can read).
        # Set it via: gh variable set AZURE_SUBSCRIPTION_ID --body <subId>
        subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }}

    - name: Install Azure CLI Resource Graph Extension
      shell: pwsh
      run: az extension add --name resource-graph --yes

    - name: Install AzLocal.UpdateManagement from PSGallery
      # v0.8.5 thin-YAML: drift detection + banner + step outputs are all
      # produced by Add-AzLocalPipelineVersionBanner (Public cmdlet).
      shell: pwsh
      id: module-version
      run: |
        $ErrorActionPreference = 'Stop'
        $installArgs = @{ Name = 'AzLocal.UpdateManagement'; Scope = 'CurrentUser'; Force = $true; AllowClobber = $true }
        if ($env:REQUIRED_MODULE_VERSION) {
            $installArgs.RequiredVersion = $env:REQUIRED_MODULE_VERSION
            Write-Host "REQUIRED_MODULE_VERSION is set - pinning install to v$($env:REQUIRED_MODULE_VERSION)."
        } else {
            Write-Host "REQUIRED_MODULE_VERSION is empty - installing the latest version from PSGallery (default fix-forward behaviour)."
        }
        Install-Module @installArgs
        Import-Module AzLocal.UpdateManagement -Force
        Add-AzLocalPipelineVersionBanner `
            -GeneratedAgainstVersion $env:GENERATED_AGAINST_MODULE_VERSION `
            -PinnedVersion $env:REQUIRED_MODULE_VERSION

    - name: Collect Fleet Connectivity Data
      # v0.8.5 thin-YAML: the inline run body (5 ARG queries via
      # Get-AzLocalFleetConnectivityStatus + per-scope severity
      # classification + ~200-line inline JUnit StringBuilder +
      # markdown summary render + 12 step outputs) has been condensed
      # into the Public cmdlet Export-AzLocalFleetConnectivityStatusReport.
      # The cmdlet writes ./reports/{7-csv,7-json,fleet-connectivity-status.xml},
      # emits the markdown summary via GITHUB_STEP_SUMMARY (using the
      # shared New-AzLocalFleetConnectivityStatusSummary renderer), and
      # sets the 12 lowercase step outputs (cluster_total, cluster_fail,
      # arc_total, arc_fail, nic_total, nic_fail, nic_all_total,
      # arb_total, arb_fail, total_failures, critical_count, warning_count).
      shell: pwsh
      id: fleet-connectivity
      env:
        INPUT_SUBSCRIPTION_FILTER: ${{ github.event.inputs.subscription_filter }}
        INSTALLED_MODULE_VERSION: ${{ steps.module-version.outputs.installed_module_version }}
      run: |
        $ErrorActionPreference = 'Stop'
        Import-Module AzLocal.UpdateManagement -Force
        $params = @{
            OutputDirectory        = './reports'
            InstalledModuleVersion = $env:INSTALLED_MODULE_VERSION
        }
        if ($env:INPUT_SUBSCRIPTION_FILTER) { $params['SubscriptionFilter'] = $env:INPUT_SUBSCRIPTION_FILTER }
        Export-AzLocalFleetConnectivityStatusReport @params

    - name: Compute Artifact Timestamp
      if: always()
      id: artifact-stamp
      shell: pwsh
      run: |
        $stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMdd_HHmmss')
        "timestamp=$stamp" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
        Write-Host "Artifact timestamp: $stamp"

    - name: Upload Fleet Connectivity Reports
      if: always()
      uses: actions/upload-artifact@v6
      with:
        name: azlocal-step.4-fleet-connectivity-status-report_${{ steps.artifact-stamp.outputs.timestamp }}
        path: ./reports/
        retention-days: 90

    # Publish JUnit test results AFTER the cmdlet (which writes the markdown
    # summary first) so the summary tables are the first thing visible.
    # list-suites/list-tests=failed keeps the expansion compact for large fleets.
    - name: Publish JUnit Diagnostic Results
      uses: dorny/test-reporter@v3
      if: always()
      with:
        name: '[JUnit Debug] Fleet Connectivity Status Report'
        path: ./reports/fleet-connectivity-status.xml
        reporter: java-junit
        fail-on-error: false
        list-suites: failed
        list-tests: failed

    # ----------------------------------------------------------------------
    # ITSM Connector (ServiceNow auto-raise on connectivity failures)
    # Opt-in (gated on inputs.raise_itsm_ticket == 'true'). Mirrors the
    # Step.9 / Step.10 wiring - reads the JUnit file produced above, uses
    # the same azurelocal-itsm.yml trigger matrix, same dedupe key.
    # ----------------------------------------------------------------------
    - name: Install powershell-yaml (ITSM config parser)
      if: ${{ github.event.inputs.raise_itsm_ticket == 'true' }}
      shell: pwsh
      run: |
        if (-not (Get-Module -ListAvailable -Name powershell-yaml)) {
            Install-Module powershell-yaml -Scope CurrentUser -Force -AllowClobber
        }

    - name: Raise ITSM tickets
      if: ${{ github.event.inputs.raise_itsm_ticket == 'true' }}
      shell: pwsh
      id: itsm
      env:
        # BEGIN-AZLOCAL-CUSTOMIZE:itsm-secrets
        # Bind your ITSM connector secrets here. Defaults match the
        # ServiceNow OAuth client_credentials naming used by azurelocal-itsm.yml.
        # Preserved by Update-AzLocalPipelineExample across module upgrades.
        ITSM_SN_INSTANCE_URL: ${{ secrets.ITSM_SN_INSTANCE_URL }}
        ITSM_SN_CLIENT_ID:    ${{ secrets.ITSM_SN_CLIENT_ID }}
        ITSM_SN_CLIENT_SECRET: ${{ secrets.ITSM_SN_CLIENT_SECRET }}
        # END-AZLOCAL-CUSTOMIZE:itsm-secrets
        INPUT_ITSM_CONFIG_PATH:  ${{ github.event.inputs.itsm_config_path }}
        INPUT_ITSM_DRY_RUN:      ${{ github.event.inputs.itsm_dry_run }}
        INPUT_ITSM_FORCE_CREATE: ${{ github.event.inputs.itsm_force_create }}
      run: |
        Import-Module AzLocal.UpdateManagement -Force

        $configPath = $env:INPUT_ITSM_CONFIG_PATH
        $dryRun     = $env:INPUT_ITSM_DRY_RUN -eq 'true'
        $force      = $env:INPUT_ITSM_FORCE_CREATE -eq 'true'

        if (-not (Test-Path $configPath)) {
            Write-Host "::warning::ITSM config not found at '$configPath' - skipping ticket creation."
            exit 0
        }

        $cfg = Get-AzLocalItsmConfig -Path $configPath

        $junitInput = './reports/fleet-connectivity-status.xml'
        if (-not (Test-Path $junitInput)) {
            Write-Host "::warning::No fleet-connectivity-status.xml found at '$junitInput' - skipping ticket creation."
            exit 0
        }

        $params = @{
            InputArtifactPath = $junitInput
            Config            = $cfg
            RunMetadata       = @{
                Platform = 'github'
                RunId    = $env:GITHUB_RUN_ID
                RunUrl   = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
                Branch   = $env:GITHUB_REF
            }
            DryRun            = $dryRun
            ForceCreate       = $force
            ExportPath        = './reports/itsm-results.csv'
            ExportJUnitPath   = './reports/itsm-results.xml'
        }

        $results = New-AzLocalIncident @params
        $results | Format-Table ClusterName, Action, TicketId, Severity -AutoSize

    - name: Upload ITSM Artefacts
      if: ${{ github.event.inputs.raise_itsm_ticket == 'true' }}
      uses: actions/upload-artifact@v6
      with:
        name: azlocal-step.4-fleet-connectivity-status-itsm-results_${{ steps.artifact-stamp.outputs.timestamp }}
        path: ./reports/itsm-*.*
        retention-days: 30
      continue-on-error: true

    - name: Publish ITSM Test Results
      if: ${{ github.event.inputs.raise_itsm_ticket == 'true' }}
      uses: dorny/test-reporter@v3
      with:
        name: ITSM Tickets (Step.4)
        path: ./reports/itsm-results.xml
        reporter: java-junit
      continue-on-error: true