fetch_and_log.yaml

name: Fetch and Log Workflow Runs

on:
#   schedule:
#     - cron: "0 */1 * * *"
  workflow_dispatch:

jobs:
  fetch-workflows:
    runs-on: ubuntu-latest
    outputs:
      workflow_ids: ${{ steps.fetch-unprocessed-run-ids.outputs.workflow_ids }}

    steps:
      - name: Get Last Processed Run ID
        id: get-last-processed-run-id
        run: |
          LAST_PROCESSED_RUN_ID=$(gh variable get --repo $GITHUB_REPOSITORY LAST_PROCESSED_RUN_ID --json value -q ".value" 2>/dev/null) || echo "Variable not found, defaulting to 0"
          LAST_PROCESSED_RUN_ID=${LAST_PROCESSED_RUN_ID:-0}
          echo "LAST_PROCESSED_RUN_ID=${LAST_PROCESSED_RUN_ID:-0}" >> $GITHUB_ENV
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Fetch Unprocessed Workflow Runs
        id: fetch-unprocessed-run-ids
        run: |
          # This script ensures that only completed GitHub workflow runs are processed while avoiding workflows that are still in progress.
          # Steps:
          # 1. Fetch the latest workflow runs from GitHub, sorted by databaseId.
          # 2. Identify the oldest incomplete workflow (first one still running).
          # 3. Return an array of workflow IDs up to (but not including) the oldest incomplete workflow ID.
          WORKFLOW_DATA=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,createdAt,displayTitle,status --jq "[.[] | select(.databaseId > $LAST_PROCESSED_RUN_ID and .displayTitle != \"Fetch and Log Workflow Runs\")] | sort_by(.databaseId)")

          OLDEST_INCOMPLETE_ID=$(echo "$WORKFLOW_DATA" | jq '[.[] | select(.status != "completed") | .databaseId] | min // empty')

          if [[ -n "$OLDEST_INCOMPLETE_ID" ]]; then
            WORKFLOW_IDS=$(echo "$WORKFLOW_DATA" | jq "[.[] | select(.status == \"completed\" and .databaseId < $OLDEST_INCOMPLETE_ID)] | [.[].databaseId] | @json")
          else
            WORKFLOW_IDS=$(echo "$WORKFLOW_DATA" | jq "[.[] | select(.status == \"completed\")] | [.[].databaseId] | @json")
          fi

          echo "Workflow runs to process: $WORKFLOW_IDS"
          echo "workflow_ids=$WORKFLOW_IDS" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        
  process-workflows:
    needs: fetch-workflows
    runs-on: ubuntu-latest
    if: ${{ needs.fetch-workflows.outputs.workflow_ids != '[]' && needs.fetch-workflows.outputs.workflow_ids != '' }}
    strategy:
      matrix:
        run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
      max-parallel: 1
    steps:
      - name: Workflow Run to Process
        run:
          echo ${{ matrix.run_id }}
     
    #   - name: Send Workflow Logs to Splunk
    #     uses: ykoer/github-workflow-splunk-logger@dev
    #     with:
    #       github_token: ${{ secrets.GITHUB_TOKEN }}
    #       splunk_url: ${{ vars.HEC_URL }}
    #       splunk_token: ${{ secrets.HEC_TOKEN }}
    #       run_id: ${{ matrix.run_id }}

      - name: Save Processed Run ID
        if: success()
        run: |
          gh variable set LAST_PROCESSED_RUN_ID --repo $GITHUB_REPOSITORY --body "${{ matrix.run_id }}"
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}


  # Add a fallback job that runs when there are no workflow Run's to process
  process-workflows-empty:
    needs: fetch-workflows
    runs-on: ubuntu-latest
    if: ${{ needs.fetch-workflows.outputs.workflow_ids == '[]' || needs.fetch-workflows.outputs.workflow_ids == '' }}
    steps:
      - name: No workflows to process
        run: echo "No new workflow runs to process"