splunkconf-backup-test.yml

name: Splunkconf backup app Integration Test

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
  workflow_dispatch:

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
  APP_NAME: "splunkconf-backup" 
  APP_DEPLOY_PATH: "/opt/splunk/etc/apps"
  OUTPUT_DIR: "/opt/splunk/var/backups"           # Replace with the specific directory to monitor
  WAIT_TIME: 300                                      # Seconds to wait after restart for file generation
  SPLUNK_PASSWORD: "Chang3d!"                         # Splunk admin password

jobs:
  test-splunk-app:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        splunk_version:
          - "10.2"
          - "10.0"
          - "9.4"
          - "9.3"
          - "9.2"
          - "9.0"

    name: "Splunk ${{ matrix.splunk_version }} : Testing backups"

    steps:
      # -------------------------------------------------------
      # Step 1: Checkout the repository
      # -------------------------------------------------------

      - name: show disk usage before checkout
        run: |
          echo "=== Disk usage ==="
          df -h .
          du -sh .
          echo ""

      - name: Checkout repository (sparse - splunkconf-backup only)
        uses: actions/checkout@v6
        with:
          sparse-checkout: |
            splunkconf-backup
          sparse-checkout-cone-mode: true


      - name: Verify checkout and show disk usage
        run: |
          echo "=== Checked out files ==="
          find . -type f | head -50
          echo ""
          echo "=== Disk usage ==="
          df -h .
          du -sh .
          echo ""
          echo "=== App directory ==="
          ls -la splunkconf-backup/ || { echo "ERROR: splunkconf-backup directory not found"; exit 1; }

      # -------------------------------------------------------
      # Step 2: Prepare the app package with test files
      # -------------------------------------------------------
      - name: Prepare app package
        run: |
          echo "=== Preparing app package ==="
          mkdir -p app_package

          # Copy the sparse-checkout app directory as the package
          cp -r splunkconf-backup app_package/${{ env.APP_NAME }}

          # Remove any git artifacts that might have been included
          rm -rf "app_package/${{ env.APP_NAME }}/.git"

          echo "=== App package contents ==="
          find app_package/ -type f | head -50
          echo ""
          du -sh app_package/
          echo "Package prepared successfully."

      # -------------------------------------------------------
      # Step 3: Install zstd (needed for .tar.zst verification)
      # -------------------------------------------------------
      - name: Install zstd
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq zstd


      # -------------------------------------------------------
      # Step 4: Start Splunk container
      # -------------------------------------------------------
      - name: Start Splunk container
        run: |
          echo "=== Starting Splunk ${{ matrix.splunk_version }} ==="

          docker run -d \
            --name splunk \
            --hostname splunk \
            -p 8000:8000 \
            -p 8089:8089 \
            -e SPLUNK_GENERAL_TERMS="--accept-sgt-current-at-splunk-com" \
            -e SPLUNK_START_ARGS="--accept-license" \
            -e SPLUNK_PASSWORD="${{ env.SPLUNK_PASSWORD }}" \
            -e TZ="Europe/Paris" \
            splunk/splunk:${{ matrix.splunk_version }}

          echo "Container started. Waiting for Splunk to become healthy..."

      # -------------------------------------------------------
      # Step 5: Wait for Splunk to be ready
      # -------------------------------------------------------
      - name: Wait for Splunk ${{ matrix.splunk_version }} to be ready
        run: |
          echo "=== Waiting for Splunk ${{ matrix.splunk_version }} to become ready ==="
          MAX_RETRIES=60
          RETRY_INTERVAL=10
          RETRIES=0

          while [ $RETRIES -lt $MAX_RETRIES ]; do
            HEALTH=$(docker inspect --format='{{.State.Health.Status}}' splunk 2>/dev/null || echo "not_found")

            if [ "$HEALTH" = "healthy" ]; then
              echo "Splunk is healthy after $((RETRIES * RETRY_INTERVAL)) seconds."
              break
            fi

            RETRIES=$((RETRIES + 1))
            echo "  Attempt ${RETRIES}/${MAX_RETRIES} - Status: ${HEALTH}. Retrying in ${RETRY_INTERVAL}s..."
            sleep $RETRY_INTERVAL
          done

          if [ "$HEALTH" != "healthy" ]; then
            echo "ERROR: Splunk did not become healthy within $((MAX_RETRIES * RETRY_INTERVAL)) seconds."
            docker logs splunk
            exit 1
          fi

          # Additional check via REST API
          HTTP_CODE=$(docker exec splunk curl -s -o /dev/null -w "%{http_code}" \
            -k -u admin:${{ env.SPLUNK_PASSWORD }} \
            https://localhost:8089/services/server/info)

          if [ "$HTTP_CODE" -eq 200 ]; then
            echo "Splunk REST API is responding (HTTP ${HTTP_CODE})."
          else
            echo "ERROR: Splunk REST API returned HTTP ${HTTP_CODE}."
            exit 1
          fi

      # -------------------------------------------------------
      # Step 6: Deploy the app into Splunk
      # -------------------------------------------------------
      - name: Deploy ${{ env.APP_NAME }} app to Splunk
        run: |
          echo "=== Deploying app to Splunk ==="

          # Copy the app package into the Splunk container
          # app is inside a git subdirectory with same folder name
          docker cp app_package/${{ env.APP_NAME }} \
            splunk:${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}

          # Create local directory and add configuration file
          echo "=== Creating local configuration ==="
          docker exec --user root splunk mkdir -p \
            ${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}/local

          docker exec --user root splunk bash -c 'cat > ${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}/local/splunkconf-backup.conf << EOF
          [settings]
          DEBUG=1
          EOF'

          echo "=== Fixing app permission, giving files to Splunk ==="
          # Fix ownership inside the container
          # run as root to be allowed to chown
          docker exec --user root splunk chown -R splunk:splunk \
            ${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}

          # Verify deployment
          echo "=== Verifying app deployment ==="
          docker exec --user splunk splunk ls -la ${{ env.APP_DEPLOY_PATH }}/${{ env.APP_NAME }}/
          echo "App deployed successfully."

      # -------------------------------------------------------
      # Step 7: Restart Splunk to load the app
      # -------------------------------------------------------
      - name: Restart Splunk
        run: |
          echo "=== Restarting Splunk to load ${{ env.APP_NAME }} app ==="

          echo "Checking id"
          docker exec splunk id
          echo "listing processes in splunk docker"
          docker exec splunk ps aux
          docker exec --user splunk splunk cat /opt/splunk/etc/splunk-launch.conf
          echo "restarting splunk as splunk user"
          docker exec --user splunk splunk /opt/splunk/bin/splunk restart 

          #          docker exec --user splunk splunk /opt/splunk/bin/splunk restart \
          #            -auth admin:${{ env.SPLUNK_PASSWORD }}

          echo "Restart command issued. Waiting for Splunk to come back..."

          # Wait for Splunk to be healthy again after restart
          MAX_RETRIES=60
          RETRY_INTERVAL=10
          RETRIES=0

          while [ $RETRIES -lt $MAX_RETRIES ]; do
            docker exec --user splunk splunk ls /opt/splunk/var
            docker exec --user splunk splunk ls -l /opt/splunk/var/backups

            HEALTH=$(docker inspect --format='{{.State.Health.Status}}' splunk 2>/dev/null || echo "not_found")

            if [ "$HEALTH" = "healthy" ]; then
              echo "Splunk is healthy again after $((RETRIES * RETRY_INTERVAL)) seconds."
              break
            fi

            RETRIES=$((RETRIES + 1))
            echo "  Attempt ${RETRIES}/${MAX_RETRIES} - Status: ${HEALTH}. Retrying in ${RETRY_INTERVAL}s..."
            sleep $RETRY_INTERVAL
          done

          echo "after restart : listing processes in splunk docker"
          docker exec splunk ps aux
          if [ "$HEALTH" != "healthy" ]; then
            echo "ERROR: Splunk did not recover after restart."
            docker logs splunk
            exit 1
          fi

      # -------------------------------------------------------
      # Step 8: Verify app is loaded in Splunk
      # -------------------------------------------------------
      - name: Verify app is loaded
        run: |
          echo "=== Verifying app is loaded in Splunk ==="

          APP_STATUS=$(docker exec --user splunk splunk /opt/splunk/bin/splunk display app ${{ env.APP_NAME }} \
            -auth admin:${{ env.SPLUNK_PASSWORD }} 2>&1 || true)

          echo "App status: ${APP_STATUS}"

          # Also check via REST API
          HTTP_CODE=$(docker exec splunk curl -s -o /dev/null -w "%{http_code}" \
            -k -u admin:${{ env.SPLUNK_PASSWORD }} \
            https://localhost:8089/servicesNS/nobody/${{ env.APP_NAME }}/configs/conf-app/ui)

          echo "REST API check for app: HTTP ${HTTP_CODE}"

      # -------------------------------------------------------
      # Step 9: Wait for file generation
      # -------------------------------------------------------
      - name: Wait for output files to be generated
        run: |
          echo "=== Waiting up to ${{ env.WAIT_TIME }} seconds for files to be generated ==="
          echo "Start time: $(date)"
          echo "Early exit condition: at least 2 backup files found"
          echo ""

          TOTAL_WAIT=${{ env.WAIT_TIME }}
          ELAPSED=0
          CHECK_INTERVAL=15
          REQUIRED_FILE_COUNT=2
          STABLE_AGE_SECONDS=5

          # Create the helper script on the runner first, then copy it into the container
          cat > /tmp/count_stable_files.sh << 'SCRIPT'
          #!/bin/bash -x
          OUTPUT_DIR="$1"
          STABLE_AGE="$2"
          NOW=$(date +%s)
          TOTAL=0
          STABLE=0
          which mktemp >&2
          which find  >&2
          mktemp --version >&2
          find --version >&2
          # Use a temp file to store the file list (avoids subshell/pipe issues)
          TMPLIST=$(mktemp /opt/splunk/filelist.XXXXXX 2>/dev/null || mktemp /tmp/filelist.XXXXXX)
          find "$OUTPUT_DIR" -type f \( -name "*.tar" -o -name "*.tar.gz" -o -name "*.tgz" -o -name "*.tar.zst" \) > "$TMPLIST" 2>/dev/null

          while IFS= read -r f; do
            [ -z "$f" ] && continue
            TOTAL=$((TOTAL + 1))
            MTIME=$(stat -c %Y "$f" 2>/dev/null || echo "0")
            AGE=$((NOW - MTIME))
            if [ "$AGE" -ge "$STABLE_AGE" ]; then
              STABLE=$((STABLE + 1))
            fi
          done < "$TMPLIST"

          rm -f "$TMPLIST"
          echo "${TOTAL} ${STABLE}"
          SCRIPT

          
          # commented . cant add executable bit here but we change permission inside docker after anyway
          # chmod +x /tmp/count_stable_files.sh


          # Copy the script into the container
          docker cp /tmp/count_stable_files.sh splunk:/opt/splunk/count_stable_files.sh
          docker exec --user root splunk chmod +x /opt/splunk/count_stable_files.sh
          echo "check bash veersion in container"
          docker exec --user splunk splunk bash --version
          echo "check bash veersion in container for ansible user"
          docker exec splunk bash --version


          while [ $ELAPSED -lt $TOTAL_WAIT ]; do
            sleep $CHECK_INTERVAL
            ELAPSED=$((ELAPSED + CHECK_INTERVAL))
            docker exec --user splunk splunk ls /opt/splunk/var
            docker exec --user splunk splunk ls -l /opt/splunk/var/backups

            # Get total and stable counts
            COUNTS=$(docker exec --user splunk splunk bash /opt/splunk/count_stable_files.sh \
              "${{ env.OUTPUT_DIR }}" "$STABLE_AGE_SECONDS" 2>/dev/null || echo "0 0")

            TOTAL_COUNT=$(echo "$COUNTS" | awk '{print $1}')
            STABLE_COUNT=$(echo "$COUNTS" | awk '{print $2}')
            # debug
            echo "debug show any error fron count stable files"
            docker exec --user splunk splunk bash -x /opt/splunk/count_stable_files.sh  "${{ env.OUTPUT_DIR }}" "$STABLE_AGE_SECONDS"
            echo "after debug"

            # Handle empty results
            TOTAL_COUNT=${TOTAL_COUNT:-0}
            STABLE_COUNT=${STABLE_COUNT:-0}


            echo "  [${ELAPSED}s/${TOTAL_WAIT}s] Total files: ${TOTAL_COUNT} | Stable (>${STABLE_AGE_SECONDS}s): ${STABLE_COUNT}"

            if [ "$STABLE_COUNT" -ge $REQUIRED_FILE_COUNT ]; then
              echo ""
              echo "  🚀 Early exit: ${STABLE_COUNT} stable file(s) found (required: ${REQUIRED_FILE_COUNT})"
              break
            fi
          done

          echo ""
          echo "End time: $(date)"
          echo "Total wait: ${ELAPSED} seconds"

          # Final count
          FINAL_COUNT=$(docker exec --user splunk splunk bash -c \
            "find ${{ env.OUTPUT_DIR }} \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.zst' \) 2>/dev/null | wc -l" \
            || echo "0")
          echo "Final file count: ${FINAL_COUNT}"

          if [ "$FINAL_COUNT" -lt $REQUIRED_FILE_COUNT ]; then
            echo "⚠️  WARNING: Only ${FINAL_COUNT} file(s) found after full wait. Expected at least ${REQUIRED_FILE_COUNT}."
          else
            echo "✅ ${FINAL_COUNT} backup file(s) detected."
          fi

      # -------------------------------------------------------
      # Step 10: Check for file creation in the output directory
      # -------------------------------------------------------
      - name: Verify files were created
        run: |
          echo "=== Checking for generated files in ${{ env.OUTPUT_DIR }} dir==="

          # List everything in the output directory
          echo "--- Full directory listing ---"
          docker exec --user splunk splunk bash -c \
            "find ${{ env.OUTPUT_DIR }} -type f -ls 2>/dev/null" || true

          # Count tar files specifically
          FILE_COUNT=$(docker exec --user splunk splunk bash -c \
            "find ${{ env.OUTPUT_DIR }} -type f \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.zst' \) 2>/dev/null | wc -l")

          echo "Total backup files found: ${FILE_COUNT}"

          if [ "$FILE_COUNT" -eq 0 ]; then
            echo "ERROR: No backup files were generated in ${{ env.OUTPUT_DIR }}."
            exit 1
          fi

          echo "SUCCESS: ${FILE_COUNT} backup file(s) found."

      # -------------------------------------------------------
      # Step 11: Download (copy) the generated tar files
      # -------------------------------------------------------
      - name: Download generated backup files from container
        run: |
          echo "=== Downloading backup files from Splunk container ==="
          mkdir -p ./test_output

          # Copy all backup files from the output directory
          docker exec --user splunk splunk bash -c \
            "find ${{ env.OUTPUT_DIR }} -type f \( -name '*.tar' -o -name '*.tar.gz' -o -name '*.tgz' -o -name '*.tar.zst'  \)" \
            | while read -r filepath; do
                filename=$(basename "$filepath")
                echo "  Downloading: ${filepath} -> ./test_output/${filename}"
                docker cp "splunk:${filepath}" "./test_output/${filename}"
              done

          echo ""
          echo "=== Downloaded files ==="
          ls -la ./test_output/

      # -------------------------------------------------------
      # Step 12: Verify tar file contents
      # -------------------------------------------------------
      - name: Verify tar file contents
        run: |
          echo "=== Verifying tar file contents ==="

          # Define expected files per backup type
          # ---- ETC backups ----
          # Filename pattern: backupconfsplunk-*-etc-targeted-*.tar.gz (or .tar.zst)
          EXPECTED_FILES_ETC=(
            "./etc/apps/splunkconf-backup/lookups/splunkconf-backup-expected.csv"
            "./etc/apps/journald_input/default/inputs.conf"
            "./etc/apps/splunk_metrics_workspace/README/workspace.conf.spec"
          )

          # ---- STATE backups ----
          # Filename pattern: backupconfsplunk-*-state-*.tar.gz (or .tar.zst)
          EXPECTED_FILES_STATE=(
            "./var/lib/splunk/fishbucket/db/"
            "./var/run/splunk/confsnapshot/"
          )

          OVERALL_RESULT=0
          ETC_FOUND=0
          STATE_FOUND=0

          # Helper function to list tar contents based on file extension
          list_tar_contents() {
            local tarfile="$1"
            case "$tarfile" in
              *.tar.zst)
                tar -I zstd -tf "$tarfile" 2>/dev/null
                ;;
              *.tar.gz|*.tgz)
                tar -tzf "$tarfile" 2>/dev/null
                ;;
              *.tar)
                tar -tf "$tarfile" 2>/dev/null
                ;;
              *)
                # Try plain, then gzip, then zstd
                tar -tf "$tarfile" 2>/dev/null || \
                tar -tzf "$tarfile" 2>/dev/null || \
                tar -I zstd -tf "$tarfile" 2>/dev/null
                ;;
            esac
          }


          # Helper function to detect backup type from filename
          get_backup_type() {
            local filename="$1"
            if echo "$filename" | grep -qE 'backupconfsplunk-.*-etc-targeted-'; then
              echo "etc"
            elif echo "$filename" | grep -qE 'backupconfsplunk-.*-state-'; then
              echo "state"
            else
              echo "unknown"
            fi
          }

          for tarfile in ./test_output/*.tar ./test_output/*.tar.gz ./test_output/*.tgz ./test_output/*.tar.zst; do
            # Skip if glob didn't match any files
            [ -e "$tarfile" ] || continue

            FILENAME=$(basename "$tarfile")
            BACKUP_TYPE=$(get_backup_type "$FILENAME")

            echo ""
            echo "============================================"
            echo "Inspecting: ${FILENAME}"
            echo "Detected backup type: ${BACKUP_TYPE}"
            echo "============================================"

            # Select the appropriate expected files list
            case "$BACKUP_TYPE" in
              etc)
                EXPECTED_FILES=("${EXPECTED_FILES_ETC[@]}")
                ETC_FOUND=1
                ;;
              state)
                EXPECTED_FILES=("${EXPECTED_FILES_STATE[@]}")
                STATE_FOUND=1
                ;;
              unknown)
                echo "  ⚠️  WARNING: Unknown backup type for file: ${FILENAME}"
                echo "  Skipping content verification for this file."
                echo "  Expected patterns:"
                echo "    - ETC:   backupconfsplunk-*-etc-targeted-*.tar.gz (or .tar.zst)"
                echo "    - STATE: backupconfsplunk-*-state-*.tar.gz (or .tar.zst)"
                continue
                ;;
            esac


            echo "--- Archive contents ---"
            TAR_CONTENTS=$(list_tar_contents "$tarfile") || {
              echo "ERROR: Unable to read file: $tarfile"
              OVERALL_RESULT=1
              continue
            }
            echo "$TAR_CONTENTS"
            echo ""

            echo "--- Checking expected files for ${BACKUP_TYPE} backup ---"
            for expected in "${EXPECTED_FILES[@]}"; do
              if echo "$TAR_CONTENTS" | grep -q "$expected"; then
                echo "  ✅ FOUND: $expected"
              else
                echo "  ❌ MISSING: $expected"
                OVERALL_RESULT=1
              fi
            done
          done


          # Verify that we found at least one of each backup type
          echo ""
          echo "============================================"
          echo "Backup type coverage check:"
          echo "============================================"

          if [ "$ETC_FOUND" -eq 1 ]; then
            echo "  ✅ ETC backup found and verified"
          else
            echo "  ❌ No ETC backup found (expected: backupconfsplunk-*-etc-targeted-*)"
            OVERALL_RESULT=1
          fi

          if [ "$STATE_FOUND" -eq 1 ]; then
            echo "  ✅ STATE backup found and verified"
          else
            echo "  ❌ No STATE backup found (expected: backupconfsplunk-*-state-*)"
            OVERALL_RESULT=1
          fi

          echo ""
          echo "============================================"
          if [ $OVERALL_RESULT -eq 0 ]; then
            echo "🎉 ALL VERIFICATIONS PASSED"
          else
            echo "💥 SOME VERIFICATIONS FAILED"
            exit 1
          fi

      # -------------------------------------------------------
      # Step 13: Check for backup failures via Splunk REST API
      # -------------------------------------------------------
      - name: Check for backup failures via Splunk REST API
        run: |
          echo "=== Searching for backup failures in Splunk ==="

          SEARCH_QUERY='search index=_internal sourcetype=splunkd source="*splunkconf-backup*" result=failure | head 50'

          SEARCH_RESULTS=$(docker exec --user splunk splunk curl -s -k \
            -u admin:${{ env.SPLUNK_PASSWORD }} \
            https://localhost:8089/services/search/jobs/export \
            -d "search=${SEARCH_QUERY}" \
            -d "output_mode=json" \
            -d "earliest_time=-1h" \
            -d "latest_time=now" \
            2>/dev/null)

          echo "=== Raw search results ==="
          echo "$SEARCH_RESULTS"
          echo ""

          EVENT_COUNT=$(echo "$SEARCH_RESULTS" | grep -c '"result"' || echo "0")

          echo "=== Failure events found: ${EVENT_COUNT} ==="

          if [ "$EVENT_COUNT" -gt 0 ]; then
            echo ""
            echo "💥 FAILURE: Found ${EVENT_COUNT} backup failure event(s) in Splunk logs!"
            echo ""
            echo "=== Failure details ==="
            echo "$SEARCH_RESULTS" | python3 -m json.tool 2>/dev/null || echo "$SEARCH_RESULTS"
            exit 1
          else
            echo "✅ No backup failures detected in Splunk logs."
          fi

      # -------------------------------------------------------
      # Step 14: Upload test artifacts (for debugging)
      # -------------------------------------------------------
      - name: Upload test artifacts
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: splunk-test-output-${{ matrix.splunk_version }}
          path: ./test_output/
          retention-days: 7

      # -------------------------------------------------------
      # Step 15: Collect Splunk logs (always, for debugging any failure)
      # -------------------------------------------------------

      - name: Collect Splunk logs
        if: always()
        run: |
          echo "=== Collecting Splunk logs ==="
          mkdir -p ./debug_logs

          # App-specific log
          echo "--- splunkconf-backup log ---"
          docker exec --user splunk splunk cat \
            /opt/splunk/var/log/splunk/splunkconf-backup.log \
            > ./debug_logs/splunkconf-backup.log 2>/dev/null || true

          # Splunk main logs
          echo "--- Splunk logs for debugging ---"
          docker exec --user splunk splunk tail -100 /opt/splunk/var/log/splunk/splunkd.log \
            > ./debug_logs/splunkd.log 2>/dev/null || true

          # Backup directory listing
          echo "--- Backup directory listing ---"
          docker exec --user splunk splunk ls -la ${{ env.OUTPUT_DIR }}/ \
            > ./debug_logs/backup-dir-listing.txt 2>/dev/null || true

          echo "=== Collected files ==="
          ls -la ./debug_logs/

          echo ""
          echo "=== splunkconf-backup.log preview ==="
          head -100 ./debug_logs/splunkconf-backup.log 2>/dev/null || echo "(empty or not found)"

          echo ""
          echo "Logs collected."

      - name: Upload debug logs
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: splunk-debug-logs-${{ matrix.splunk_version }}
          path: ./debug_logs/
          retention-days: 7

      # -------------------------------------------------------
      # Step 16: Cleanup
      # -------------------------------------------------------
      - name: Cleanup Splunk container
        if: always()
        run: |
          docker stop splunk 2>/dev/null || true
          docker rm splunk 2>/dev/null || true
          echo "Cleanup complete."