fetch_and_log.yaml

name: Fetch and Log Workflow Runs

on:
#   schedule:
#     - cron: "0 */1 * * *"
  workflow_dispatch:

jobs:
  fetch-workflows:
    runs-on: ubuntu-latest
    outputs:
      workflow_ids: ${{ steps.fetch-unprocessed-run-ids.outputs.workflow_ids }}

    steps:
      - name: Get Last Processed Run ID
        id: get-last-processed-run-id
        run: |
          LAST_PROCESSED_RUN_ID=$(gh variable get --repo $GITHUB_REPOSITORY LAST_PROCESSED_RUN_ID --json value -q ".value" 2>/dev/null) || echo "Variable not found, defaulting to 0"
          LAST_PROCESSED_RUN_ID=${LAST_PROCESSED_RUN_ID:-0}
          echo "LAST_PROCESSED_RUN_ID=${LAST_PROCESSED_RUN_ID:-0}" >> $GITHUB_ENV
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}

      - name: Fetch Unprocessed Workflow Runs
        id: fetch-unprocessed-run-ids
        run: |
          # TODO: We need to address an important issue where workflow runs that are currently in progress may be included.
          # These should be excluded from processing, and we should revisit them in the next run once they have finished.
          WORKFLOW_IDS=$(gh run list --repo $GITHUB_REPOSITORY --limit 100 --json databaseId,createdAt,displayTitle --jq "[.[] | select(.databaseId > $LAST_PROCESSED_RUN_ID and .displayTitle != \"Fetch and Log Workflow Runs\")] | sort_by(.databaseId) | [.[].databaseId] | @json")
          echo "Workflow runs to process: $WORKFLOW_IDS"
          echo "workflow_ids=$WORKFLOW_IDS" >> $GITHUB_OUTPUT
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        
  process-workflows:
    needs: fetch-workflows
    runs-on: ubuntu-latest
    if: ${{ needs.fetch-workflows.outputs.workflow_ids != '[]' && needs.fetch-workflows.outputs.workflow_ids != '' }}
    strategy:
      matrix:
        run_id: ${{ fromJson(needs.fetch-workflows.outputs.workflow_ids || '[]') }}
      max-parallel: 1
    steps:
      - name: Workflow Run to Process
        run:
          echo ${{ matrix.run_id }}
     
    #   - name: Send Workflow Logs to Splunk
    #     uses: ykoer/github-workflow-splunk-logger@dev
    #     with:
    #       github_token: ${{ secrets.GITHUB_TOKEN }}
    #       splunk_url: ${{ vars.HEC_URL }}
    #       splunk_token: ${{ secrets.HEC_TOKEN }}
    #       run_id: ${{ matrix.run_id }}

      - name: Save Processed Run ID
        if: success()
        run: |
          gh variable set LAST_PROCESSED_RUN_ID --repo $GITHUB_REPOSITORY --body "${{ matrix.run_id }}"
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}


  # Add a fallback job that runs when there are no workflow Run's to process
  process-workflows-empty:
    needs: fetch-workflows
    runs-on: ubuntu-latest
    if: ${{ needs.fetch-workflows.outputs.workflow_ids == '[]' || needs.fetch-workflows.outputs.workflow_ids == '' }}
    steps:
      - name: No workflows to process
        run: echo "No new workflow runs to process"