nightly-parallel.yml

name: Nightly Parallel Integration Tests

on:
  push:
    branches:
      - feat/nightly-parallel-integration-tests 
  schedule:
    - cron: '30 18 * * *'  # 12:00 AM IST (UTC+5:30 = 15:30 UTC)
  workflow_dispatch:
    inputs:
      reason:
        description: 'Reason for manual trigger'
        required: false
        default: 'Manual run'

concurrency:
  group: nightly-parallel
  cancel-in-progress: true

permissions:
  contents: read

env:
  GO_COVERAGE_PKGS: "github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers"

jobs:
  # ─────────────────────────────────────────────────────────────────────────────
  # Job A: Scan all integration test files and detect any tests not yet assigned
  #        to a named matrix group, so they fall through to the catch-all run.
  # ─────────────────────────────────────────────────────────────────────────────
  validate-test-coverage:
    runs-on: ubuntu-latest
    outputs:
      uncovered_tests: ${{ steps.find-uncovered.outputs.uncovered_tests }}
      has_uncovered: ${{ steps.find-uncovered.outputs.has_uncovered }}
    steps:
      - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0

      - name: Find tests not covered by any named group
        id: find-uncovered
        run: |
          # Combined regex of every pattern used across the 12 named matrix groups.
          # Any test whose name does NOT match this will land in the catch-all group.
          # Built via concatenation so every line stays at ≥10-space YAML indentation.
          CP="TestCreateScan|TestScanCreate|TestScansE2E|TestFastScan"
          CP="${CP}|TestLightQueries|TestRecommendedExclusions|TestScansUpdateProjectGroups"
          CP="${CP}|TestBrokenLinkScan|TestScanWithPolicy|TestCreateAsyncScan|TestScanGLReport"
          CP="${CP}|TestContainerEngineScansE2E|TestScanListWith|TestScanShowRequired"
          CP="${CP}|TestRequiredScanId|TestScaResolver|TestInvalidSource|TestIncrementalScan"
          CP="${CP}|TestBranchPrimary|TestCancelScan|TestScanTimeout|TestScanWorkflow|TestScanLog"
          CP="${CP}|TestPartialScan|TestFailedScan|TestRunKics|TestRunSca|TestScaRealtime"
          CP="${CP}|TestScanType|TestValidateScan|TestScanGenerating|TestResult|TestCodeBashing"
          CP="${CP}|TestRiskManagement|TestCreateQueryDescription|TestPR|TestPreReceive"
          CP="${CP}|TestPre_Receive|TestProject|TestCreateEmptyProject|TestCreateAlreadyExisting"
          CP="${CP}|TestCreateWithInvalid|TestCreateProjectWhen|TestGetProject|TestSastUpdate"
          CP="${CP}|TestGetAndUpdate|TestPredicate|TestTriage|TestScaUpdate|TestRunGetBfl"
          CP="${CP}|TestContainerScan|TestContainerImage|TestContainersRealtime|TestIacRealtime"
          CP="${CP}|TestSecrets_Realtime|TestOssRealtime|TestScanASCA|TestExecuteASCA"
          CP="${CP}|TestEngineNameResolution|TestAuth|TestFailProxy|TestLoadConfiguration"
          CP="${CP}|TestSetConfig|TestMain|TestRootVersion|TestSetLog|Test_Download|TestGitHub"
          CP="${CP}|TestGitLab|TestBitbucket|TestBitBucket|TestAzure|TestHooksPreCommit"
          CP="${CP}|TestGetLearnMore|TestImport|TestGetTenant|TestMaskSecrets|TestFailedMask"
          CP="${CP}|TestScaRemediation|TestKicsRemediation|TestTelemetry|Test_Handle|TestChat"
          COVERED_PATTERNS="${CP}"

          ALL_TESTS=$(grep -rh "^func Test" test/integration/*_test.go \
            | sed 's/func \(Test[^(]*\).*/\1/')

          UNCOVERED=""
          while IFS= read -r test; do
            [ -z "$test" ] && continue
            if ! echo "$test" | grep -qE "$COVERED_PATTERNS"; then
              UNCOVERED="${UNCOVERED:+${UNCOVERED}|}${test}"
            fi
          done <<< "$ALL_TESTS"

          if [ -n "$UNCOVERED" ]; then
            echo "Uncovered tests detected: $UNCOVERED"
            {
              echo "uncovered_tests=$UNCOVERED"
              echo "has_uncovered=true"
            } >> "$GITHUB_OUTPUT"
          else
            echo "All tests are covered by named groups."
            {
              echo "uncovered_tests="
              echo "has_uncovered=false"
            } >> "$GITHUB_OUTPUT"
          fi

  # ─────────────────────────────────────────────────────────────────────────────
  # Job B: Run each test group in parallel across 13 matrix entries.
  #        The 13th entry (uncovered) is a dynamic catch-all driven by Job A.
  # ─────────────────────────────────────────────────────────────────────────────
  integration-tests:
    needs: validate-test-coverage
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          # 1 ── Scan Creation (heavy; needs pre-run cleanup)
          - name: scan-create
            label: "Scan Creation"
            run_pattern: "TestCreateScan|TestScanCreate|TestScansE2E|TestFastScan|TestLightQueries|TestRecommendedExclusions|TestScansUpdateProjectGroups|TestBrokenLinkScan|TestScanWithPolicy|TestCreateAsyncScan|TestScanGLReport|TestContainerEngineScansE2E"
            timeout: "90m"
            needs_precommit: "false"
            run_cleandata: "true"

          # 2 ── Scan Operations (list, show, logs, kics, sca; needs pre-run cleanup)
          - name: scan-ops
            label: "Scan Operations"
            run_pattern: "TestScanListWith|TestScanShowRequired|TestRequiredScanId|TestScaResolver|TestInvalidSource|TestIncrementalScan|TestBranchPrimary|TestCancelScan|TestScanTimeout|TestScanWorkflow|TestScanLog|TestPartialScan|TestFailedScan|TestRunKics|TestRunSca|TestScaRealtime|TestScanType|TestValidateScan|TestScanGenerating"
            timeout: "90m"
            needs_precommit: "false"
            run_cleandata: "true"

          # 3 ── Results & Reports
          - name: results
            label: "Results & Reports"
            run_pattern: "TestResult|TestCodeBashing|TestRiskManagement|TestCreateQueryDescription"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 4 ── PR Decoration
          - name: pr-decoration
            label: "PR Decoration"
            run_pattern: "TestPR|TestPreReceive|TestPre_Receive"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 5 ── Projects (needs pre-run cleanup)
          - name: projects
            label: "Projects"
            run_pattern: "TestProject|TestCreateEmptyProject|TestCreateAlreadyExisting|TestCreateWithInvalid|TestCreateProjectWhen|TestGetProject"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "true"

          # 6 ── Predicates & BFL
          - name: predicates
            label: "Predicates & BFL"
            run_pattern: "TestSastUpdate|TestGetAndUpdate|TestPredicate|TestTriage|TestScaUpdate|TestRunGetBfl"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 7 ── Container Tests
          - name: containers
            label: "Container Tests"
            run_pattern: "TestContainerScan|TestContainerImage"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 8 ── Realtime Scanning (ASCA, IAC, Secrets, OSS)
          - name: realtime
            label: "Realtime Scanning"
            run_pattern: "TestContainersRealtime|TestIacRealtime|TestSecrets_Realtime|TestOssRealtime|TestScanASCA|TestExecuteASCA|TestEngineNameResolution"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 9 ── Auth & Config
          - name: auth-config
            label: "Auth & Config"
            run_pattern: "TestAuth|TestFailProxy|TestLoadConfiguration|TestSetConfig"
            timeout: "30m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 10 ── Root & Logs
          - name: root-logs
            label: "Root & Logs"
            run_pattern: "TestMain|TestRootVersion|TestSetLog|Test_Download"
            timeout: "30m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 11 ── SCM Rate Limit & User Count
          - name: scm-tests
            label: "SCM Rate Limit & User Count"
            run_pattern: "TestGitHub|TestGitLab|TestBitbucket|TestBitBucket|TestAzure"
            timeout: "60m"
            needs_precommit: "false"
            run_cleandata: "false"

          # 12 ── Miscellaneous (pre-commit required)
          - name: misc
            label: "Miscellaneous"
            run_pattern: "TestHooksPreCommit|TestGetLearnMore|TestImport|TestGetTenant|TestMaskSecrets|TestFailedMask|TestScaRemediation|TestKicsRemediation|TestTelemetry|Test_Handle|TestChat"
            timeout: "60m"
            needs_precommit: "true"
            run_cleandata: "false"

          # 13 ── Catch-All (dynamic; pattern injected at runtime from Job A output)
          - name: uncovered
            label: "Catch-All (Uncovered)"
            run_pattern: "__UNCOVERED__"
            timeout: "90m"
            needs_precommit: "false"
            run_cleandata: "false"

    env:
      CX_BASE_URI: ${{ secrets.CX_BASE_URI }}
      CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }}
      CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }}
      CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }}
      CX_AST_USERNAME: ${{ secrets.CX_AST_USERNAME }}
      CX_AST_PASSWORD: ${{ secrets.CX_AST_PASSWORD }}
      CX_APIKEY: ${{ secrets.CX_APIKEY }}
      CX_TENANT: ${{ secrets.CX_TENANT }}
      CX_SCAN_SSH_KEY: ${{ secrets.CX_SCAN_SSH_KEY }}
      CX_ORIGIN: "cli-tests"
      PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      PROXY_HOST: localhost
      PROXY_PORT: 3128
      PROXY_USERNAME: ${{ secrets.PROXY_USER }}
      PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
      PR_GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      PR_GITHUB_NAMESPACE: "checkmarx"
      PR_GITHUB_REPO_NAME: "ast-cli"
      PR_GITHUB_NUMBER: 983
      PR_GITLAB_TOKEN: ${{ secrets.PR_GITLAB_TOKEN }}
      PR_GITLAB_NAMESPACE: ${{ secrets.PR_GITLAB_NAMESPACE }}
      PR_GITLAB_REPO_NAME: ${{ secrets.PR_GITLAB_REPO_NAME }}
      PR_GITLAB_PROJECT_ID: ${{ secrets.PR_GITLAB_PROJECT_ID }}
      PR_GITLAB_IID: ${{ secrets.PR_GITLAB_IID }}
      AZURE_ORG: ${{ secrets.AZURE_ORG }}
      AZURE_PROJECT: ${{ secrets.AZURE_PROJECT }}
      AZURE_REPOS: ${{ secrets.AZURE_REPOS }}
      AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }}
      AZURE_PR_NUMBER: 1
      BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }}
      BITBUCKET_REPOS: ${{ secrets.BITBUCKET_REPOS }}
      BITBUCKET_USERNAME: ${{ secrets.BITBUCKET_USERNAME }}
      BITBUCKET_PASSWORD: ${{ secrets.BITBUCKET_PASSWORD }}
      GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
      GITHUB_ACTOR: ${{ github.actor }}
      PR_BITBUCKET_TOKEN: ${{ secrets.PR_BITBUCKET_TOKEN }}
      PR_BITBUCKET_NAMESPACE: "AstSystemTest"
      PR_BITBUCKET_REPO_NAME: "cliIntegrationTest"
      PR_BITBUCKET_ID: 1

    steps:
      - name: Checkout repository
        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0

      - name: Set up Go
        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4
        with:
          go-version-file: go.mod

      - name: Build binary
        run: go build -o ./bin/cx ./cmd

      - name: Install gocovmerge
        run: go install github.com/wadey/gocovmerge@latest

      - name: Install pre-commit
        if: matrix.needs_precommit == 'true'
        run: |
          pip install pre-commit
          pre-commit install

      - name: Start Squid proxy
        run: |
          docker run \
            --name squid \
            -d \
            -p 3128:3128 \
            -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \
            -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \
            ubuntu/squid:5.2-22.04_beta

      - name: Download ScaResolver
        run: |
          wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz
          tar -xzvf ScaResolver-linux64.tar.gz -C /tmp
          rm -rf ScaResolver-linux64.tar.gz

      - name: Pre-test cleanup (${{ matrix.label }})
        if: matrix.run_cleandata == 'true'
        run: go test -v github.com/checkmarx/ast-cli/test/cleandata

      - name: Run integration tests (${{ matrix.label }})
        if: matrix.name != 'uncovered' || needs.validate-test-coverage.outputs.has_uncovered == 'true'
        run: |
          set -euo pipefail

          # Resolve the -run pattern: catch-all uses Job A output; named groups use matrix field.
          if [ "${{ matrix.name }}" = "uncovered" ]; then
            RUN_PATTERN="${{ needs.validate-test-coverage.outputs.uncovered_tests }}"
          else
            RUN_PATTERN="${{ matrix.run_pattern }}"
          fi

          COVER_FILE="cover-${{ matrix.name }}.out"

          run_tests() {
            local pattern="$1" outfile="$2" logfile="$3" timeout_val="$4"
            go test \
              -tags integration \
              -v \
              -timeout "${timeout_val}" \
              -coverpkg "${{ env.GO_COVERAGE_PKGS }}" \
              -coverprofile "${outfile}" \
              -run "${pattern}" \
              github.com/checkmarx/ast-cli/test/integration 2>&1 | tee "${logfile}" || true
          }

          echo "::group::Attempt 1 — ${{ matrix.label }}"
          run_tests "$RUN_PATTERN" "$COVER_FILE" "test_output.log" "${{ matrix.timeout }}"
          echo "::endgroup::"

          FAILED=$(grep -E "^--- FAIL: " test_output.log | awk '{print $3}' | paste -sd '|' - || true)

          # ── Retry 1 ──────────────────────────────────────────────────────────
          if [ -n "$FAILED" ]; then
            echo "::warning::Retry 1 for ${{ matrix.label }}: $FAILED"
            COVER_R1="cover-${{ matrix.name }}-r1.out"
            echo "::group::Attempt 2 — ${{ matrix.label }}"
            run_tests "$FAILED" "$COVER_R1" "retry1_output.log" "30m"
            echo "::endgroup::"

            if [ -f "$COVER_R1" ]; then
              gocovmerge "$COVER_FILE" "$COVER_R1" > merged.out
              mv merged.out "$COVER_FILE"
              rm -f "$COVER_R1"
            fi

            FAILED2=$(grep -E "^--- FAIL: " retry1_output.log | awk '{print $3}' | paste -sd '|' - || true)

            # ── Retry 2 ────────────────────────────────────────────────────────
            if [ -n "$FAILED2" ]; then
              echo "::warning::Retry 2 for ${{ matrix.label }}: $FAILED2"
              COVER_R2="cover-${{ matrix.name }}-r2.out"
              echo "::group::Attempt 3 — ${{ matrix.label }}"
              run_tests "$FAILED2" "$COVER_R2" "retry2_output.log" "30m"
              echo "::endgroup::"

              if [ -f "$COVER_R2" ]; then
                gocovmerge "$COVER_FILE" "$COVER_R2" > merged.out
                mv merged.out "$COVER_FILE"
                rm -f "$COVER_R2"
              fi

              FINAL_FAILED=$(grep -E "^--- FAIL: " retry2_output.log | awk '{print $3}' || true)
              if [ -n "$FINAL_FAILED" ]; then
                echo "::error::Tests still failing after 2 retries in ${{ matrix.label }}: $FINAL_FAILED"
                exit 1
              fi
            fi
          fi

          echo "All ${{ matrix.label }} tests passed."

      - name: Upload coverage artifact
        if: always() && (matrix.name != 'uncovered' || needs.validate-test-coverage.outputs.has_uncovered == 'true')
        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4
        with:
          name: coverage-${{ matrix.name }}
          path: cover-${{ matrix.name }}.out
          retention-days: 7
          if-no-files-found: warn

      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4
        with:
          name: test-logs-${{ matrix.name }}
          path: |
            test_output.log
            retry1_output.log
            retry2_output.log
          retention-days: 7
          if-no-files-found: ignore

      - name: Stop Squid proxy
        if: always()
        run: docker stop squid && docker rm squid || true

  # ─────────────────────────────────────────────────────────────────────────────
  # Job C: Download all per-group coverage files, merge them, check >= 75%,
  #        upload the HTML report, and run a final project cleanup.
  # ─────────────────────────────────────────────────────────────────────────────
  merge-coverage:
    needs: integration-tests
    runs-on: ubuntu-latest
    if: always()
    env:
      CX_BASE_URI: ${{ secrets.CX_BASE_URI }}
      CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }}
      CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }}
      CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }}
      CX_AST_USERNAME: ${{ secrets.CX_AST_USERNAME }}
      CX_AST_PASSWORD: ${{ secrets.CX_AST_PASSWORD }}
      CX_APIKEY: ${{ secrets.CX_APIKEY }}
      CX_TENANT: ${{ secrets.CX_TENANT }}
    steps:
      - uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0

      - name: Set up Go
        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4
        with:
          go-version-file: go.mod

      - name: Install gocovmerge
        run: go install github.com/wadey/gocovmerge@latest

      - name: Download all coverage artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: coverage-*
          merge-multiple: true

      - name: Merge coverage profiles
        run: |
          COVER_FILES=$(ls cover-*.out 2>/dev/null | tr '\n' ' ')
          if [ -z "$COVER_FILES" ]; then
            echo "::error::No coverage files found — all groups may have been skipped or failed."
            exit 1
          fi
          echo "Merging: $COVER_FILES"
          # shellcheck disable=SC2086
          gocovmerge $COVER_FILES > cover.out
          go tool cover -html=cover.out -o coverage.html

      - name: Check coverage threshold (>= 75%)
        run: |
          CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}')
          EXPECTED=75
          echo "Total coverage: ${CODE_COV}%"
          var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED'" }')
          if [ "$var" -eq 1 ]; then
            echo "::error::Coverage too low: ${CODE_COV}% (required >= ${EXPECTED}%)"
            exit 1
          else
            echo "Coverage OK: ${CODE_COV}%"
          fi

      - name: Upload merged HTML report
        if: always()
        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4
        with:
          name: coverage-report-merged
          path: coverage.html
          retention-days: 7

      - name: Post-run project cleanup
        run: go test -v github.com/checkmarx/ast-cli/test/cleandata

  # ─────────────────────────────────────────────────────────────────────────────
  # Job D: Write a GitHub Actions Job Summary when any job in the chain fails.
  #        No external service required — everything goes to GITHUB_STEP_SUMMARY.
  # ─────────────────────────────────────────────────────────────────────────────
  notify-on-failure:
    needs: [integration-tests, merge-coverage]
    runs-on: ubuntu-latest
    if: failure()
    steps:
      - name: Write failure summary
        env:
          INTEGRATION_RESULT: ${{ toJson(needs.integration-tests) }}
          MERGE_RESULT: ${{ toJson(needs.merge-coverage) }}
        run: |
          cat >> "$GITHUB_STEP_SUMMARY" << 'SUMMARY'
          ## Nightly Parallel Integration Tests — FAILED

          | Field | Value |
          |-------|-------|
          | Run | ${{ github.run_id }} |
          | Triggered by | ${{ github.event_name }} |
          | Branch | ${{ github.ref_name }} |
          | Commit | ${{ github.sha }} |
          | Schedule | 12:00 AM IST (18:30 UTC) |
          SUMMARY

          printf '\n### integration-tests result\n```json\n%s\n```\n' "$INTEGRATION_RESULT" \
            >> "$GITHUB_STEP_SUMMARY"
          printf '\n### merge-coverage result\n```json\n%s\n```\n' "$MERGE_RESULT" \
            >> "$GITHUB_STEP_SUMMARY"

          cat >> "$GITHUB_STEP_SUMMARY" << 'SUMMARY'

          ### Next Steps
          1. Click each failed matrix group in the job list above to inspect its logs.
          2. Download the `test-logs-<group>` artifact for the full `go test` output.
          3. Retry a specific group manually via **Run workflow** (`workflow_dispatch`).
          4. If the failure is consistent, open an issue referencing this run.
          SUMMARY