diff --git a/.ci/requirements.txt b/.ci/requirements.txt index 45eb253548496..6c6abcada3eaf 100644 --- a/.ci/requirements.txt +++ b/.ci/requirements.txt @@ -1,3 +1,4 @@ junitparser==3.2.0 google-cloud-storage==3.3.0 PyGithub==2.8.1 +defusedxml==0.7.1 diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index f8d029ae537a2..4d2342ec53b5d 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -142,20 +142,18 @@ jobs: run: cmake --install build - name: Test adapter specific - env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - # These tests cause timeouts on CI - LIT_FILTER_OUT: "(adapters/level_zero/memcheck.test|adapters/level_zero/v2/deferred_kernel_memcheck.test)" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-adapter + uses: ./devops/actions/run-tests/ur/adapter-specific + with: + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-adapter-specific' # Don't run adapter specific tests when building multiple adapters if: ${{ matrix.adapter.other_name == '' }} - name: Test adapters - env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-conformance + uses: ./devops/actions/run-tests/ur/adapters + with: + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-conformance' - name: Debug CI platform information if: ${{ always() }} diff --git a/devops/actions/run-tests/ur/adapter-specific/action.yml b/devops/actions/run-tests/ur/adapter-specific/action.yml new file mode 100644 index 0000000000000..40e0d2e618821 --- /dev/null +++ b/devops/actions/run-tests/ur/adapter-specific/action.yml @@ -0,0 +1,122 @@ +name: 'Run UR Adapter Specific Tests' + +inputs: + build_dir: + required: false + default: 'build' + artifact_name: + description: 'Name for uploaded test artifacts (logs, XML)' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Run adapter specific tests + id: run_tests + continue-on-error: true + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + ZE_ENABLE_LOADER_DEBUG_TRACE: 1 + LIT_OPTS: "--show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped --xunit-xml-output adapter_results.xml" + LIT_FILTER_OUT: "(adapters/level_zero/memcheck.test|adapters/level_zero/v2/deferred_kernel_memcheck.test)" + run: | + if [[ "$BUILD_DIR" =~ [^a-zA-Z0-9/_-] ]] || [[ "$BUILD_DIR" =~ \.\. ]]; then + echo "::error::Invalid build_dir: contains disallowed characters or path traversal" + exit 1 + fi + + echo "::group::Show Full Test Adapter Specific Logs" + # Write directly to file to avoid SIGPIPE from GitHub Actions closing stdout + # (output is too large without --succinct, causing GitHub to close pipe) + cmake --build "$BUILD_DIR" -j $(($(nproc)/3)) -- check-unified-runtime-adapter > adapter_tests.log 2>&1 + EXIT_CODE=$? + + # Optionally show full LIT output including test lists from summary + # Set SHOW_FULL_TEST_LISTS=0 to filter out duplicate test lists (default: show all for debugging) + if [ "${SHOW_FULL_TEST_LISTS:-1}" = "1" ]; then + echo "=== Full test execution log (including LIT summary lists) ===" || true + cat adapter_tests.log || true + else + # Filtered: Show only summary without duplicate test lists + # (LIT summary with asterisks is filtered out to avoid duplication with collapsed lists below) + echo "=== Test execution log (filtered - lists shown in collapsed sections below) ===" || true + sed '/^\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*/,$d' adapter_tests.log || true + fi + echo "::endgroup::" + + # Only fail if cmake actually failed, not if test failed (continue-on-error handles that) + # Check that log was created successfully + if [ ! -f adapter_tests.log ] || [ ! -s adapter_tests.log ]; then + echo "::error::Test execution failed - no log generated" + exit 1 + fi + + exit $EXIT_CODE + + - name: Report adapter specific failures + if: steps.run_tests.outcome != 'success' + shell: bash + run: | + python3 devops/scripts/ur_test_summary.py extract-errors adapter_tests.log | tee -a $GITHUB_STEP_SUMMARY + exit 1 + + - name: Show adapter specific test statistics and lists + if: ${{ always() }} + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + run: | + if [ ! -f adapter_tests.log ]; then + echo "::warning::Adapter specific test log not found - no tests may be available for this adapter" + echo "## Adapter Specific Tests Not Available" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "No adapter-specific tests were found or executed for this configuration." >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + if [ ! -s adapter_tests.log ]; then + echo "::warning::Adapter specific test log is empty" + echo "## Adapter Specific Tests Log Empty" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The adapter-specific test log exists but is empty." >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + # Find actual XML file path (wildcard doesn't work in Python) + XML_FILE=$(find "$BUILD_DIR/test/adapters" -name "adapter_results.xml" 2>/dev/null | head -1) + if [ -z "$XML_FILE" ]; then + XML_FILE="" + fi + + python3 devops/scripts/ur_test_summary.py show-summary adapter_tests.log "" "$XML_FILE" | tee -a $GITHUB_STEP_SUMMARY + + - name: Prepare adapter specific test artifacts + if: ${{ always() && inputs.artifact_name != '' }} + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + run: | + mkdir -p adapter_artifacts + + # Copy log file + if [ -f adapter_tests.log ]; then + cp adapter_tests.log adapter_artifacts/ + fi + + # Copy XML file + # Use find since we have wildcard + FOUND_XML=$(find "$BUILD_DIR/test/adapters" -name "adapter_results.xml" 2>/dev/null | head -1) + if [ -n "$FOUND_XML" ]; then + cp "$FOUND_XML" adapter_artifacts/adapter_results.xml + fi + + - name: Upload adapter specific test artifacts + if: ${{ always() && inputs.artifact_name != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.5.2 + with: + name: ${{ inputs.artifact_name }} + path: adapter_artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/devops/actions/run-tests/ur/adapters/action.yml b/devops/actions/run-tests/ur/adapters/action.yml new file mode 100644 index 0000000000000..f4dadee997470 --- /dev/null +++ b/devops/actions/run-tests/ur/adapters/action.yml @@ -0,0 +1,127 @@ +name: 'Run UR Adapter Conformance Tests' + +inputs: + build_dir: + required: false + default: 'build' + artifact_name: + description: 'Name for uploaded test artifacts (logs, XML)' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Run conformance tests + id: run_tests + continue-on-error: true + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + ZE_ENABLE_LOADER_DEBUG_TRACE: 1 + LIT_OPTS: "--show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped --xunit-xml-output conformance_results.xml" + run: | + if [[ "$BUILD_DIR" =~ [^a-zA-Z0-9/_-] ]] || [[ "$BUILD_DIR" =~ \.\. ]]; then + echo "::error::Invalid build_dir: contains disallowed characters or path traversal" + exit 1 + fi + + # Write directly to file to avoid SIGPIPE from GitHub Actions closing stdout + # (output is too large without --succinct, causing GitHub to close pipe) + cmake --build "$BUILD_DIR" -j $(($(nproc)/3)) -- check-unified-runtime-conformance > conformance_tests.log 2>&1 + EXIT_CODE=$? + + # Only fail if cmake actually failed, not if test failed (continue-on-error handles that) + # Check that log was created successfully + if [ ! -f conformance_tests.log ] || [ ! -s conformance_tests.log ]; then + echo "::error::Test execution failed - no log generated" + exit 1 + fi + + exit $EXIT_CODE + + - name: Report conformance failures + if: steps.run_tests.outcome != 'success' + shell: bash + run: | + python3 devops/scripts/ur_test_summary.py extract-errors conformance_tests.log | tee -a $GITHUB_STEP_SUMMARY + exit 1 + + - name: List all conformance tests + if: ${{ always() }} + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + run: | + > all_conformance_tests.txt + binary_count=0 + + # Use process substitution to avoid subshell issues with pipe + while IFS= read -r binary; do + binary_count=$((binary_count + 1)) + + # Capture output to temp file to avoid SIGPIPE + temp_output=$(mktemp) + "$binary" --gtest_list_tests > "$temp_output" 2>&1 || true + + # Filter and append to all_conformance_tests.txt + grep -v "^$" "$temp_output" >> all_conformance_tests.txt 2>/dev/null || true + rm -f "$temp_output" + done < <(find "$BUILD_DIR/test/conformance" -name "*-test" -type f -executable 2>/dev/null) + + test_count=$(wc -l < all_conformance_tests.txt 2>/dev/null || echo 0) + echo "Summary: Processed $binary_count binaries, collected $test_count test definition lines" >&2 + + - name: Show conformance test statistics and lists + if: ${{ always() }} + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + run: | + if [ ! -f conformance_tests.log ]; then + echo "Log file not found" + exit 0 + fi + + # XML file is generated by LIT in the test directory + XML_FILE="$BUILD_DIR/test/conformance/conformance_results.xml" + if [ ! -f "$XML_FILE" ]; then + echo "Note: XML file not found at $XML_FILE, skipped test names will not be available" + XML_FILE="" + fi + + python3 devops/scripts/ur_test_summary.py show-summary conformance_tests.log all_conformance_tests.txt "$XML_FILE" | tee -a $GITHUB_STEP_SUMMARY + + - name: Prepare conformance test artifacts + if: ${{ always() && inputs.artifact_name != '' }} + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + run: | + mkdir -p conformance_artifacts + + # Copy log file + if [ -f conformance_tests.log ]; then + cp conformance_tests.log conformance_artifacts/ + fi + + # Copy XML file + XML_PATH="$BUILD_DIR/test/conformance/conformance_results.xml" + if [ -f "$XML_PATH" ]; then + cp "$XML_PATH" conformance_artifacts/ + else + # Try to find it elsewhere + FOUND_XML=$(find "$BUILD_DIR" -name "conformance_results.xml" 2>/dev/null | head -1) + if [ -n "$FOUND_XML" ]; then + cp "$FOUND_XML" conformance_artifacts/conformance_results.xml + fi + fi + + - name: Upload conformance test artifacts + if: ${{ always() && inputs.artifact_name != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.5.2 + with: + name: ${{ inputs.artifact_name }} + path: conformance_artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/devops/scripts/generate_test_summary.sh b/devops/scripts/generate_test_summary.sh new file mode 100755 index 0000000000000..959d0271a14b8 --- /dev/null +++ b/devops/scripts/generate_test_summary.sh @@ -0,0 +1,247 @@ +#!/bin/bash + +# Generate test summary from LIT output log +# Usage: generate_test_summary.sh + +set -euo pipefail + +LOG_FILE="${1:-}" +TEST_TYPE="${2:-Tests}" + +if [ -z "$LOG_FILE" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +# Basic input validation +if [[ "$LOG_FILE" =~ [\;\&\|\`\$] ]]; then + echo "ERROR: Invalid characters in log file path" >&2 + exit 1 +fi + +# Sanitize TEST_TYPE to prevent HTML/markdown injection +# Allow only alphanumeric, spaces, hyphens, underscores +TEST_TYPE=$(echo "$TEST_TYPE" | tr -cd '[:alnum:][:space:]-_' | tr -s ' ') + +if [ ! -f "$LOG_FILE" ]; then + printf '### %s Summary\n\n' "$TEST_TYPE" + printf 'Log file not found: %s\n\n' "$LOG_FILE" + exit 0 +fi + +# Check if file is empty +if [ ! -s "$LOG_FILE" ]; then + printf '### %s Summary\n\n' "$TEST_TYPE" + printf 'Log file is empty: %s\n\n' "$LOG_FILE" + exit 0 +fi + +# Parse LIT summary statistics +parse_summary_stats() { + awk ' + /^Total Discovered Tests:/ { gsub(/[^0-9]/, ""); if ($0 != "") print "TOTAL:" $0 } + /^ Skipped:/ { match($0, /[0-9]+/); if (RSTART > 0) print "SKIPPED_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Passed[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "PASSED_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Failed[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "FAILED_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Unsupported[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "UNSUPPORTED_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Unresolved[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "UNRESOLVED_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Timeout[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "TIMEOUT_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Expected Passes[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "XFAIL_COUNT:" substr($0, RSTART, RLENGTH) } + /^ Unexpected Passes[ :]/ { match($0, /[0-9]+/); if (RSTART > 0) print "XPASS_COUNT:" substr($0, RSTART, RLENGTH) } + ' "$LOG_FILE" +} + +# Parse test results from log +parse_results() { + awk ' + function extract_test_name(prefix) { + line = $0 + sub("^" prefix ": ", "", line) + sub(/ \([0-9]+ of [0-9]+\)$/, "", line) + return line + } + + /^PASS: / { print "PASS:" extract_test_name("PASS") } + /^FAIL: / { print "FAIL:" extract_test_name("FAIL") } + /^XFAIL: / { print "XFAIL:" extract_test_name("XFAIL") } + /^XPASS: / { print "XPASS:" extract_test_name("XPASS") } + /^UNSUPPORTED: / { print "UNSUPPORTED:" extract_test_name("UNSUPPORTED") } + /^TIMEOUT: / { print "TIMEOUT:" extract_test_name("TIMEOUT") } + /^UNRESOLVED: / { print "UNRESOLVED:" extract_test_name("UNRESOLVED") } + /^SKIPPED: / { print "SKIPPED:" extract_test_name("SKIPPED") } + ' "$LOG_FILE" +} + +# Parse results with error handling +if ! results=$(parse_results); then + printf 'ERROR: Failed to parse test results from %s\n' "$LOG_FILE" >&2 + exit 1 +fi + +# Initialize arrays for test results +declare -a timeout_tests=() +declare -a fail_tests=() +declare -a xpass_tests=() +declare -a xfail_tests=() +declare -a unsup_tests=() +declare -a unres_tests=() +declare -a skip_tests=() +declare -a pass_tests=() + +# Populate arrays from parsed results +while IFS=: read -r status test_name; do + case "$status" in + TIMEOUT) timeout_tests+=("$test_name") ;; + FAIL) fail_tests+=("$test_name") ;; + XPASS) xpass_tests+=("$test_name") ;; + XFAIL) xfail_tests+=("$test_name") ;; + UNSUPPORTED) unsup_tests+=("$test_name") ;; + UNRESOLVED) unres_tests+=("$test_name") ;; + SKIPPED) skip_tests+=("$test_name") ;; + PASS) pass_tests+=("$test_name") ;; + esac +done <<< "$results" + +# Parse summary statistics from LIT output (source of truth for counts) +# These override array-based counts where available, especially for skipped tests +# which don't appear in verbose output +summary_stats=$(parse_summary_stats) +total_tests="" +summary_skip_count="" +summary_pass_count="" +summary_fail_count="" +summary_unsup_count="" +summary_unres_count="" +summary_timeout_count="" +summary_xfail_count="" +summary_xpass_count="" + +while IFS=: read -r key value; do + case "$key" in + TOTAL) total_tests="$value" ;; + SKIPPED_COUNT) summary_skip_count="$value" ;; + PASSED_COUNT) summary_pass_count="$value" ;; + FAILED_COUNT) summary_fail_count="$value" ;; + UNSUPPORTED_COUNT) summary_unsup_count="$value" ;; + UNRESOLVED_COUNT) summary_unres_count="$value" ;; + TIMEOUT_COUNT) summary_timeout_count="$value" ;; + XFAIL_COUNT) summary_xfail_count="$value" ;; + XPASS_COUNT) summary_xpass_count="$value" ;; + esac +done <<< "$summary_stats" + +# Calculate counts from array lengths +timeout_count=${#timeout_tests[@]} +fail_count=${#fail_tests[@]} +xpass_count=${#xpass_tests[@]} +xfail_count=${#xfail_tests[@]} +unsup_count=${#unsup_tests[@]} +unres_count=${#unres_tests[@]} +skip_count=${#skip_tests[@]} +pass_count=${#pass_tests[@]} + +# Use summary counts where available (more accurate, especially for skipped tests) +[ -n "$summary_timeout_count" ] && timeout_count=$summary_timeout_count +[ -n "$summary_fail_count" ] && fail_count=$summary_fail_count +[ -n "$summary_xpass_count" ] && xpass_count=$summary_xpass_count +[ -n "$summary_xfail_count" ] && xfail_count=$summary_xfail_count +[ -n "$summary_unsup_count" ] && unsup_count=$summary_unsup_count +[ -n "$summary_unres_count" ] && unres_count=$summary_unres_count +[ -n "$summary_skip_count" ] && skip_count=$summary_skip_count +[ -n "$summary_pass_count" ] && pass_count=$summary_pass_count + +# Generate summary +printf '### %s Summary\n' "$TEST_TYPE" + +# Show overall statistics +if [ -n "$total_tests" ] && [ "$total_tests" -gt 0 ]; then + printf '\n**Total Discovered Tests:** %s\n' "$total_tests" +fi +if [ "$pass_count" -gt 0 ]; then + printf '**Passed:** %d\n' "$pass_count" +fi +if [ "$skip_count" -gt 0 ]; then + printf '**Skipped:** %d\n' "$skip_count" +fi +if [ "$fail_count" -gt 0 ]; then + printf '**Failed:** %d\n' "$fail_count" +fi +if [ "$xfail_count" -gt 0 ]; then + printf '**Expected Failures:** %d\n' "$xfail_count" +fi +if [ "$xpass_count" -gt 0 ]; then + printf '**Unexpected Passes:** %d\n' "$xpass_count" +fi +if [ "$unsup_count" -gt 0 ]; then + printf '**Unsupported:** %d\n' "$unsup_count" +fi +if [ "$timeout_count" -gt 0 ]; then + printf '**Timeouts:** %d\n' "$timeout_count" +fi +if [ "$unres_count" -gt 0 ]; then + printf '**Unresolved:** %d\n' "$unres_count" +fi +printf '\n' + +# Detailed test lists by category (collapsed) + +# Helper function to print test list in code block +print_test_list() { + local -n tests_ref=$1 + printf '\n```\n' + for test in "${tests_ref[@]}"; do + printf '%s\n' "$test" + done + printf '```\n' +} + +# Define display order and labels +declare -a category_order=("TIMEOUT" "FAIL" "XPASS" "XFAIL" "UNSUPPORTED" "UNRESOLVED" "SKIPPED" "PASS") +declare -A category_labels=( + ["TIMEOUT"]="Timeout Tests" + ["FAIL"]="Failed Tests" + ["XPASS"]="Unexpected Passed Tests (XPASS)" + ["XFAIL"]="Expected Failures (XFAIL)" + ["UNSUPPORTED"]="Unsupported Tests" + ["UNRESOLVED"]="Unresolved Tests" + ["SKIPPED"]="Skipped Tests" + ["PASS"]="Passed Tests" +) +declare -A category_arrays=( + ["TIMEOUT"]="timeout_tests" + ["FAIL"]="fail_tests" + ["XPASS"]="xpass_tests" + ["XFAIL"]="xfail_tests" + ["UNSUPPORTED"]="unsup_tests" + ["UNRESOLVED"]="unres_tests" + ["SKIPPED"]="skip_tests" + ["PASS"]="pass_tests" +) +declare -a open_by_default=("TIMEOUT" "FAIL" "XPASS") + +for cat in "${category_order[@]}"; do + declare -n test_array="${category_arrays[$cat]}" + count=${#test_array[@]} + + if [ "$count" -gt 0 ]; then + # Check if this category should be open by default + open_attr="" + for open_cat in "${open_by_default[@]}"; do + if [ "$cat" = "$open_cat" ]; then + open_attr=" open" + break + fi + done + + printf '\n\n%s (%d)\n\n' "$open_attr" "${category_labels[$cat]}" "$count" + print_test_list test_array + printf '\n\n' + fi +done + +# Exit with error if there are failures or timeouts +if [ "$fail_count" -gt 0 ] || [ "$timeout_count" -gt 0 ]; then + exit 1 +fi + +exit 0 diff --git a/devops/scripts/ur_test_summary.py b/devops/scripts/ur_test_summary.py new file mode 100755 index 0000000000000..4e17940db7649 --- /dev/null +++ b/devops/scripts/ur_test_summary.py @@ -0,0 +1,839 @@ +#!/usr/bin/env python3 +""" +Unified Runtime test summary processing for GitHub Actions CI. + +This script processes LIT test output logs to: +- Extract error details from failures/timeouts +- Show statistics and collapsed test lists for GitHub Actions +""" + +import sys +import re +from typing import List, Dict, Tuple +from pathlib import Path + +# Use defusedxml for secure XML parsing to prevent XML attacks +# This is required - install via: pip install defusedxml +import defusedxml.ElementTree as ET + + +def read_log_file(log_path: str) -> List[str]: + """Read log file and return lines.""" + try: + with open(log_path, "r", encoding="utf-8", errors="replace") as f: + return f.readlines() + except FileNotFoundError: + print(f"Error: Log file not found: {log_path}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error reading log file: {e}", file=sys.stderr) + sys.exit(1) + + +def extract_error_details(lines: List[str]) -> List[str]: + """ + Extract error details from FAIL/TIMEOUT entries. + + Stops before test list sections to avoid duplication + (lists are shown in collapsed sections). + """ + result = [] + in_error = False + + # Test list headers that mark the end of error details + list_headers_pattern = re.compile( + r"^(Passed|Unsupported|Failed|Expectedly Failed|" + r"Timed Out|Unexpectedly Passed|Unresolved) Tests \(" + ) + + for line in lines: + # Start capturing on FAIL/TIMEOUT + if re.match(r"^(FAIL|TIMEOUT):", line): + in_error = True + + # Stop at test list headers + if in_error and list_headers_pattern.match(line): + break + + if in_error: + result.append(line) + + return result + + +def extract_statistics(lines: List[str]) -> List[str]: + """ + Extract test statistics from LIT summary. + + Matches lines like: + - Total Discovered Tests: 123 + - Passed: 100 (81.30%) + - Failed: 2 (1.63%) + etc. + """ + stats_pattern = re.compile( + r"^\s*(Total Discovered|Expected Passes|Expectedly Failed|" + r"Excluded|Unsupported|Skipped|Passed|Passed With Retry|" + r"Failed|Timed Out|Unexpectedly Passed|Unresolved)(\s+Tests)?\s*:" + ) + + result = [] + for line in lines: + if stats_pattern.match(line): + result.append(line) + + return result + + +def extract_time_summary(lines: List[str]) -> Dict[str, List[str]]: + """ + Extract test timing information from LIT output. + + LIT with --time-tests flag produces: + Slowest Tests: + ---------------------------------------------------------------------- + 4.16s: test_name + 0.10s: test_name2 + ... + + Tests Times: + ---------------------------------------------------------------------- + [Range] :: [Percentage] :: [Count] + ---------------------------------------------------------------------- + [0.00s, 1.00s) :: [========] :: [10/20] + ... + ---------------------------------------------------------------------- + ******************** + + Returns dictionary with 'slowest' and 'histogram' keys containing line lists. + """ + result = {"slowest": [], "histogram": []} + current_section = None + skip_next_hr = False + + for line in lines: + stripped = line.strip() + + # Detect section starts + if stripped == "Slowest Tests:": + current_section = "slowest" + skip_next_hr = True + continue + elif stripped == "Tests Times:" or stripped == "Test Times:": + current_section = "histogram" + skip_next_hr = True + continue + + # Skip horizontal rule after section header + if skip_next_hr and stripped.startswith("---"): + skip_next_hr = False + continue + + # Collect lines for current section + if current_section == "slowest": + # Slowest section ends at empty line or next section + if not stripped: + current_section = None + elif not stripped.startswith("---"): + result["slowest"].append(line.rstrip()) + elif current_section == "histogram": + # Histogram section ends at: + # - Empty line + # - Line with only asterisks (section separator) + # - Line not starting with '[' and not only dashes + if not stripped: + current_section = None + elif stripped.replace("*", "") == "": + # Line contains only asterisks - end of section + current_section = None + elif stripped.startswith("[") or stripped.replace("-", "") == "": + # Valid histogram line (header, data row, or separator) + result["histogram"].append(line.rstrip()) + else: + # Something else - end of histogram section + current_section = None + + return result + + +def extract_skipped_from_xml(xml_path: str) -> List[str]: + """ + Extract skipped test names from LIT xunit XML output. + + LIT generates XML with --xunit-xml-output flag: + + + + + + + + + Returns list of skipped test names in format: classname.name + Excludes tests with "Test not selected" message (those are excluded, not skipped). + """ + if not xml_path: + return [] + + if not Path(xml_path).exists(): + print(f"Note: XML file not found: {xml_path}", file=sys.stderr) + return [] + + skipped = [] + try: + tree = ET.parse(xml_path) + root = tree.getroot() + + # Iterate through all testsuites and testcases + for testsuite in root.findall(".//testsuite"): + for testcase in testsuite.findall("testcase"): + # Check if testcase has child element + skipped_elem = testcase.find("skipped") + if skipped_elem is not None: + # Exclude tests with "Test not selected" - those are excluded, not skipped + message = skipped_elem.get("message", "") + if "Test not selected" in message: + continue + + classname = testcase.get("classname", "") + name = testcase.get("name", "") + + # Format: classname.name (match GoogleTest format) + if classname and name: + full_name = f"{classname}.{name}" + skipped.append(full_name) + elif name: + skipped.append(name) + + except ET.ParseError as e: + print(f"Warning: Failed to parse XML file {xml_path}: {e}", file=sys.stderr) + except Exception as e: + print(f"Warning: Error reading XML file {xml_path}: {e}", file=sys.stderr) + + if xml_path and Path(xml_path).exists(): + print(f"Note: Found {len(skipped)} skipped tests in XML file", file=sys.stderr) + + return skipped + + +def extract_excluded_from_xml(xml_path: str) -> List[str]: + """ + Extract excluded test names from LIT xunit XML output. + + LIT marks excluded tests as skipped with specific message: + + + + + + + + + Returns list of excluded test names in format: classname.name + """ + if not xml_path: + return [] + + if not Path(xml_path).exists(): + print(f"Note: XML file not found: {xml_path}", file=sys.stderr) + return [] + + excluded = [] + try: + tree = ET.parse(xml_path) + root = tree.getroot() + + # Iterate through all testsuites and testcases + for testsuite in root.findall(".//testsuite"): + for testcase in testsuite.findall("testcase"): + # Check if testcase has child with "Test not selected" message + skipped_elem = testcase.find("skipped") + if skipped_elem is not None: + message = skipped_elem.get("message", "") + if "Test not selected" in message: + classname = testcase.get("classname", "") + name = testcase.get("name", "") + + # Format: classname.name (match GoogleTest format) + if classname and name: + full_name = f"{classname}.{name}" + excluded.append(full_name) + elif name: + excluded.append(name) + + except ET.ParseError as e: + print(f"Warning: Failed to parse XML file {xml_path}: {e}", file=sys.stderr) + except Exception as e: + print(f"Warning: Error reading XML file {xml_path}: {e}", file=sys.stderr) + + if xml_path and Path(xml_path).exists(): + print( + f"Note: Found {len(excluded)} excluded tests in XML file", file=sys.stderr + ) + + return excluded + + +def extract_skipped_from_gtest(lines: List[str]) -> List[str]: + """ + Extract skipped test names from GoogleTest output. + + GoogleTest shows skipped tests in summary section: + [ SKIPPED ] 3 tests, listed below: + [ SKIPPED ] TestName1 + [ SKIPPED ] TestName2 + ... + + Returns list of skipped test names. + """ + skipped = [] + in_summary = False + + for line in lines: + # Start collecting after "[ SKIPPED ] N tests, listed below:" + if re.match(r"^\[ SKIPPED \] \d+ tests, listed below:", line): + in_summary = True + continue + + # Stop at "X SKIPPED TESTS" line or empty line after list + if in_summary and (re.match(r"^ *\d+ SKIPPED TESTS", line) or not line.strip()): + break + + # Collect test names from summary + if in_summary: + match = re.match(r"^\[ SKIPPED \] (.+)$", line) + if match: + test_name = match.group(1).strip() + skipped.append(test_name) + + return skipped + + +def extract_unsupported_from_lit_inline(lines: List[str]) -> List[str]: + """ + Extract unsupported/skipped test names from LIT inline output. + + During test execution, LIT prints: + UNSUPPORTED: TestName (reason) + SKIP: TestName + + This is used for GoogleTest format where LIT doesn't generate + "Unsupported Tests (N):" summary list, but shows "Skipped" in statistics. + + Returns list of unsupported/skipped test names (deduplicated). + """ + unsupported = [] + seen = set() + # Pattern: "UNSUPPORTED: test_name" or "SKIP: test_name" + unsupported_pattern = re.compile(r"^(UNSUPPORTED|SKIP):\s+(.+?)(?:\s+\(|$)") + + for line in lines: + match = unsupported_pattern.match(line) + if match: + test_name = match.group(2).strip() + if test_name not in seen: + seen.add(test_name) + unsupported.append(test_name) + + return unsupported + + +def extract_test_lists( + lines: List[str], +) -> Tuple[Dict[str, List[str]], Dict[str, int]]: + """ + Extract categorized test lists from LIT summary. + + Looks for sections like: + Passed Tests (N): + test name 1 + test name 2 + + Returns tuple of: + - Dictionary with category names as keys and test lists as values + - Dictionary with category names as keys and declared counts from headers + """ + categories = {} + declared_counts = {} + current_category = None + current_tests = [] + current_declared_count = 0 + + # Pattern: "Category Tests (N):" + # Match any word followed by "Tests (N):" to catch unknown categories + category_pattern = re.compile(r"^([A-Za-z ]+) Tests \((\d+)\):") + + for line in lines: + # Check for category header + match = category_pattern.match(line) + if match: + # Save previous category + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + # Start new category + current_category = match.group(1) + current_declared_count = int(match.group(2)) + current_tests = [] + continue + + # If we're in a category + if current_category: + # Empty line ends the category + if not line.strip(): + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + current_category = None + current_tests = [] + current_declared_count = 0 + else: + # Add test to current category (strip leading whitespace) + test_name = line.strip() + if test_name: + current_tests.append(test_name) + + # Save last category + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + return categories, declared_counts + + +def parse_gtest_list(all_tests_file: str) -> List[str]: + """ + Parse --gtest_list_tests output to get full test names. + + Format: + TestSuite. + TestName1/Param # comment + TestName2 + + Returns list of full test names: TestSuite.TestName1/Param + """ + if not Path(all_tests_file).exists(): + return [] + + tests = [] + current_suite = None + + # Patterns to ignore (errors, warnings, GoogleTest verification messages) + ignore_patterns = [ + "Error:", + "Warning:", + "Actual:", + "Expected:", + "Value of:", + "Failure", + "UninstantiatedParameterizedTestSuite", + "/__w/", # File paths from error messages + "No platforms", + ] + + with open(all_tests_file, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.rstrip() + if not line: + continue + + # Skip error messages and warnings + if any(pattern in line for pattern in ignore_patterns): + continue + + # Skip GoogleTestVerification suite (contains only warnings, not real tests) + if line == "GoogleTestVerification.": + current_suite = None + continue + + # Suite line ends with '.' + if line.endswith(".") and not line.startswith(" "): + current_suite = line + # Test name (indented with exactly 2 spaces) + elif line.startswith(" ") and current_suite: + # Remove leading whitespace and comments + test_name = line.strip().split(" #")[0].strip() + if test_name: + full_name = f"{current_suite}{test_name}" + tests.append(full_name) + + return tests + + +def show_statistics_and_lists(lines: List[str], all_tests_file: str = None) -> None: + """ + Extract categorized test lists from LIT summary. + + Returns dict like: + { + 'Passed': ['test1', 'test2', ...], + 'Failed': ['test3'], + ... + } + """ + categories = {} + current_category = None + current_tests = [] + + # Pattern for category headers: "Passed Tests (123):" + # Note: LIT does not generate "Skipped Tests" - only Unsupported and Expectedly Failed + header_pattern = re.compile( + r"^(Passed|Unsupported|Failed|Expectedly Failed|" + r"Timed Out|Unexpectedly Passed|Unresolved) Tests \((\d+)\):" + ) + + for line in lines: + match = header_pattern.match(line) + if match: + # Save previous category + if current_category: + categories[current_category] = current_tests + + # Start new category + current_category = match.group(1) + current_tests = [] + elif current_category: + # Empty line ends the category + if not line.strip(): + categories[current_category] = current_tests + current_category = None + current_tests = [] + else: + # Add test to current category (strip leading whitespace) + test_name = line.strip() + if test_name: + current_tests.append(test_name) + + # Save last category + if current_category: + categories[current_category] = current_tests + + return categories + + +def show_statistics_and_lists( + lines: List[str], all_tests_file: str = None, xml_file: str = None +) -> None: + """ + Show test statistics and collapsed test lists for GitHub Actions. + + Args: + lines: Log file lines + all_tests_file: Optional file with --gtest_list_tests output + xml_file: Optional LIT xunit XML output file + + Output format: + - Statistics section (always visible) + - Collapsed sections for each test category + """ + # Extract statistics + stats = extract_statistics(lines) + if stats: + print("=== Test Statistics ===") + for stat in stats: + print(stat.rstrip()) + print() + + # Validate totals match sum of categories + total_discovered = None + for stat in stats: + if "Total Discovered" in stat: + match = re.search(r"(\d+)", stat) + if match: + total_discovered = int(match.group(1)) + break + + # Extract test lists in collapsed sections (LIT format) + test_lists, declared_counts = extract_test_lists(lines) + + # Track if we display skipped/excluded separately (for validation later) + displayed_skipped_count = 0 + displayed_excluded_count = 0 + stats_skipped_count = None + stats_excluded_count = None + + # Extract skipped count from statistics + for stat in stats: + if "Skipped:" in stat or "Unsupported:" in stat: + match = re.search(r"(\d+)", stat) + if match: + stats_skipped_count = int(match.group(1)) + break + + # Handle Skipped/Unsupported tests + # Check if LIT already extracted them to logs (with --show-skipped flag) + skipped_from_log = test_lists.get("Skipped", test_lists.get("Unsupported", [])) + declared_skipped_count = declared_counts.get( + "Skipped", declared_counts.get("Unsupported", None) + ) + + if skipped_from_log and declared_skipped_count: + # We have skipped tests from log - verify count matches header + actual_count = len(skipped_from_log) + + if actual_count == declared_skipped_count: + # Count matches declared count in log header - use log data + displayed_skipped_count = actual_count + print(f"::group::Skipped Tests ({actual_count})") + for test in skipped_from_log: + print(test) + print("::endgroup::") + else: + # Mismatch between header and actual lines - fallback to XML + skipped_xml = extract_skipped_from_xml(xml_file) + if skipped_xml: + count = len(skipped_xml) + displayed_skipped_count = count + print(f"::group::Skipped Tests ({count})") + print( + f"Note: Using XML data (log header claimed {declared_skipped_count}, but found {actual_count} lines)." + ) + print() + for test in skipped_xml: + print(test) + print("::endgroup::") + else: + # No XML fallback - use log data anyway with warning + displayed_skipped_count = actual_count + print(f"::group::Skipped Tests ({actual_count})") + print( + f"Warning: Log header claimed {declared_skipped_count} skipped, but found {actual_count} lines." + ) + print() + for test in skipped_from_log: + print(test) + print("::endgroup::") + + # Remove from test_lists to avoid duplicate display + test_lists.pop("Skipped", None) + test_lists.pop("Unsupported", None) + + elif stats_skipped_count: + # No skipped in log, but statistics show some - try XML + skipped_xml = extract_skipped_from_xml(xml_file) + if skipped_xml: + count = len(skipped_xml) + displayed_skipped_count = count + print(f"::group::Skipped Tests ({count})") + for test in skipped_xml: + print(test) + print("::endgroup::") + else: + # No XML data either + print(f"::group::Skipped Tests ({stats_skipped_count})") + print("Warning: Could not extract individual skipped test names.") + print( + f"Statistics show {stats_skipped_count} skipped tests, but they are not available in the output." + ) + print("::endgroup::") + + # Handle Excluded tests similarly + # BUT: only if they're not already in test_lists (avoid duplicates) + if "Excluded" not in test_lists: + # Extract from LIT xunit XML output + excluded_xml = extract_excluded_from_xml(xml_file) + + # Extract excluded count from statistics + for stat in stats: + if "Excluded:" in stat: + match = re.search(r"(\d+)", stat) + if match: + stats_excluded_count = int(match.group(1)) + break + + if excluded_xml: + count = len(excluded_xml) + displayed_excluded_count = count + + print(f"::group::Excluded Tests ({count})") + for test in excluded_xml: + print(test) + print("::endgroup::") + elif stats_excluded_count: + # Statistics show excluded but we couldn't extract the list + print(f"::group::Excluded Tests ({stats_excluded_count})") + print(f"Warning: Could not extract individual excluded test names.") + print( + f"Statistics show {stats_excluded_count} excluded tests, but they are not available in the output." + ) + print("::endgroup::") + + # Show remaining test categories from LIT format + for category, tests in test_lists.items(): + count = len(tests) + if count > 0: + print(f"::group::{category} Tests ({count})") + for test in tests: + print(test) + print("::endgroup::") + + # Validate that total discovered matches sum of all categories + if total_discovered: + # Sum from test_lists + any skipped/excluded we displayed separately + # BUT: Don't double-count if they're already in test_lists + sum_categories = sum(len(tests) for tests in test_lists.values()) + + # Only add displayed counts if we displayed them separately + if displayed_skipped_count > 0 and "Skipped" not in test_lists: + sum_categories += displayed_skipped_count + if displayed_excluded_count > 0 and "Excluded" not in test_lists: + sum_categories += displayed_excluded_count + + if total_discovered != sum_categories: + print() + print( + f"::warning::Test count mismatch: Total Discovered = {total_discovered}, but sum of all categories = {sum_categories}" + ) + print( + f"Warning: {total_discovered - sum_categories} tests are unaccounted for." + ) + print( + f"This may indicate tests in unexpected categories or parsing issues." + ) + print(f"Categories found: {', '.join(test_lists.keys())}") + if displayed_skipped_count > 0: + print( + f"(Plus {displayed_skipped_count} skipped tests displayed separately)" + ) + if displayed_excluded_count > 0: + print( + f"(Plus {displayed_excluded_count} excluded tests displayed separately)" + ) + print() + + # Extract and display test timing information if available + time_info = extract_time_summary(lines) + + # Extract Testing Time from logs + testing_time = None + for line in lines: + if line.strip().startswith("Testing Time:"): + testing_time = line.strip() + break + + if time_info["slowest"] or time_info["histogram"] or testing_time: + print("::group::Test Timing Summary") + + if testing_time: + print(testing_time) + print() + + if time_info["slowest"]: + print("Slowest Tests:") + print("-" * 70) + for line in time_info["slowest"]: + print(line) + print() + + if time_info["histogram"]: + print("Test Times Distribution:") + print("-" * 70) + for line in time_info["histogram"]: + print(line) + + print("::endgroup::") + + +def main(): + """Main CLI interface.""" + if len(sys.argv) < 2: + print( + "Usage: ur_test_summary.py [all_tests_file] [xml_file]", + file=sys.stderr, + ) + print("\nCommands:", file=sys.stderr) + print( + " extract-errors - Extract FAIL/TIMEOUT error details", + file=sys.stderr, + ) + print( + " show-summary [all_tests] [xml] - Show statistics and collapsed test lists", + file=sys.stderr, + ) + print( + "\nArguments:", + file=sys.stderr, + ) + print( + " log - LIT test output log file", + file=sys.stderr, + ) + print( + " all_tests - Optional: output from --gtest_list_tests (for computed skipped)", + file=sys.stderr, + ) + print( + " xml - Optional: LIT xunit XML output (--xunit-xml-output)", + file=sys.stderr, + ) + sys.exit(1) + + command = sys.argv[1] + + # All commands require log file + if len(sys.argv) < 3: + print( + f"Error: Command '{command}' requires a log file argument", + file=sys.stderr, + ) + sys.exit(1) + + log_file = sys.argv[2] + + # Security: Validate path to prevent traversal attacks + if ".." in log_file or log_file.startswith("/"): + print( + f"Error: Invalid log file path (absolute paths and '..' not allowed): {log_file}", + file=sys.stderr, + ) + sys.exit(1) + + if not Path(log_file).exists(): + print(f"Error: Log file not found: {log_file}", file=sys.stderr) + sys.exit(1) + + lines = read_log_file(log_file) + + if command == "extract-errors": + result = extract_error_details(lines) + for line in result: + print(line, end="") + + elif command == "show-summary": + # Optional: all tests file for GoogleTest format + all_tests_file = sys.argv[3] if len(sys.argv) > 3 else None + # Empty string means not provided (from bash "") + if all_tests_file == "": + all_tests_file = None + if all_tests_file and ( + ".." in all_tests_file or all_tests_file.startswith("/") + ): + print( + f"Error: Invalid all_tests file path: {all_tests_file}", + file=sys.stderr, + ) + sys.exit(1) + + # Optional: XML file from LIT xunit output + xml_file = sys.argv[4] if len(sys.argv) > 4 else None + # Empty string means not provided (from bash "") + if xml_file == "": + xml_file = None + # Validate: reject path traversal but allow absolute paths (BUILD_DIR can be absolute) + if xml_file and ".." in xml_file: + print( + f"Error: Invalid XML file path (path traversal): {xml_file}", + file=sys.stderr, + ) + sys.exit(1) + + show_statistics_and_lists(lines, all_tests_file, xml_file) + + else: + print(f"Error: Unknown command: {command}", file=sys.stderr) + print("Valid commands: extract-errors, show-summary", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 8ab7543fba9b0..7a54be7b8247f 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -87,7 +87,8 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped USES_TERMINAL ) else() diff --git a/unified-runtime/test/adapters/CMakeLists.txt b/unified-runtime/test/adapters/CMakeLists.txt index 6101a11d2a1f2..93555023e8a66 100644 --- a/unified-runtime/test/adapters/CMakeLists.txt +++ b/unified-runtime/test/adapters/CMakeLists.txt @@ -5,6 +5,9 @@ add_custom_target(check-unified-runtime-adapter) +# CI validation test suite (intentional failures/timeouts for testing CI logging) +add_subdirectory(ci-validation) + if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL) add_subdirectory(cuda) endif() diff --git a/unified-runtime/test/adapters/ci-validation/CMakeLists.txt b/unified-runtime/test/adapters/ci-validation/CMakeLists.txt new file mode 100644 index 0000000000000..1566c77b940cb --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright (C) 2025 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# CI validation test suite - intentionally generates different test outcomes +# to validate CI logging and categorization + +add_ur_lit_testsuite(adapter-ci-validation) + +# NOTE: CI validation tests are now integrated into adapter-specific test suites +# (e.g., cuda/ci-validation/) to appear together in test results. +# This standalone suite is kept for compatibility but not added as dependency. diff --git a/unified-runtime/test/adapters/ci-validation/README.md b/unified-runtime/test/adapters/ci-validation/README.md new file mode 100644 index 0000000000000..043e0dda70779 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/README.md @@ -0,0 +1,53 @@ +# CI Validation Test Suite + +This test suite is designed to validate CI logging and test categorization in GitHub Actions workflows. + +## Purpose + +These tests intentionally generate different test outcomes to verify that our CI workflow correctly: +- Extracts and displays test statistics +- Categorizes tests into appropriate groups (Passed, Failed, Skipped, etc.) +- Properly displays collapsed sections in GitHub Actions Step Summary + +## Test Scenarios + +| Test File | Expected Outcome | CI Category | +|-----------|------------------|-------------| +| `test_pass.test` | Pass | Passed Tests | +| `test_fail.test` | Fail | Failed Tests | +| `test_unsupported.test` | Skip | Unsupported Tests | +| `test_xfail.test` | Expected Fail | Expectedly Failed Tests | +| `test_unexpected_pass.test` | Unexpected Pass | Unexpectedly Passed Tests | +| `test_timeout.test` | Timeout | Timed Out Tests | + +## Running Locally + +```bash +# Run all validation tests +cd build +cmake --build . --target check-unified-runtime-adapter-ci-validation + +# Or use LIT directly with timeout +python3 llvm/utils/lit/lit.py \ + --show-pass --show-unsupported --show-xfail --succinct \ + --timeout 5 \ + unified-runtime/test/adapters/ci-validation +``` + +## Expected Statistics + +When run on a Linux system, you should see approximately: +- Total Discovered Tests: 6 +- Passed: 1 +- Failed: 1 +- Unsupported: 1 +- Expectedly Failed: 1 +- Unexpectedly Passed: 1 +- Timed Out: 1 + +## Notes + +- `test_timeout.test` requires LIT timeout to be set (e.g., `--timeout 5`) +- `test_unsupported.test` is marked UNSUPPORTED on linux/windows (i.e., all platforms) +- `test_xfail.test` uses XFAIL marker - shows as "Expectedly Failed Tests" in LIT output +- These tests should NOT be run in production CI on every commit (too slow due to timeout test) diff --git a/unified-runtime/test/adapters/ci-validation/lit.local.cfg.py b/unified-runtime/test/adapters/ci-validation/lit.local.cfg.py new file mode 100644 index 0000000000000..91ac6fd6d0ff3 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/lit.local.cfg.py @@ -0,0 +1,10 @@ +""" +Copyright (C) 2025 Intel Corporation + +Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +See LICENSE.TXT +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +""" + +config.suffixes = [".test"] diff --git a/unified-runtime/test/adapters/ci-validation/test_fail.test b/unified-runtime/test/adapters/ci-validation/test_fail.test new file mode 100644 index 0000000000000..f5824a12b3bd5 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_fail.test @@ -0,0 +1,9 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// RUN: not python3 %S/test_helper.py fail | FileCheck --check-prefix=CHECK-FAIL %s +// CHECK-FAIL: Test failed: expected 42, got 41 + +// This test demonstrates a failing test scenario. +// It should appear in "Failed Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_helper.py b/unified-runtime/test/adapters/ci-validation/test_helper.py new file mode 100755 index 0000000000000..8d5f7740615da --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_helper.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +""" +Copyright (C) 2025 Intel Corporation +Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +Helper script for CI test validation - generates different test outcomes +""" + +import sys +import time + + +def main(): + if len(sys.argv) < 2: + print("Usage: test_helper.py ") + sys.exit(1) + + test_type = sys.argv[1] + + if test_type == "pass": + print("Test passed: result = 42") + sys.exit(0) + elif test_type == "fail": + print("Test failed: expected 42, got 41") + sys.exit(1) + elif test_type == "timeout": + print("Test starting infinite loop...") + while True: + time.sleep(1) + elif test_type == "crash": + print("Test about to crash...") + sys.exit(137) # SIGKILL + else: + print(f"Unknown test type: {test_type}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/unified-runtime/test/adapters/ci-validation/test_pass.test b/unified-runtime/test/adapters/ci-validation/test_pass.test new file mode 100644 index 0000000000000..889f6af45b0b4 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_pass.test @@ -0,0 +1,9 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// RUN: python3 %S/test_helper.py pass | FileCheck --check-prefix=CHECK-PASS %s +// CHECK-PASS: Test passed: result = 42 + +// This test demonstrates a passing test scenario. +// It should appear in "Passed Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_timeout.test b/unified-runtime/test/adapters/ci-validation/test_timeout.test new file mode 100644 index 0000000000000..3d0e8a85a39c7 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_timeout.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// RUN: python3 %S/test_helper.py timeout | FileCheck --check-prefix=CHECK-TIMEOUT %s +// CHECK-TIMEOUT: Test starting infinite loop + +// This test intentionally times out (infinite loop). +// It should appear in "Timed Out Tests" category in CI logs. +// Note: LIT timeout must be configured (e.g., --timeout 5) to catch this. diff --git a/unified-runtime/test/adapters/ci-validation/test_unexpected_pass.test b/unified-runtime/test/adapters/ci-validation/test_unexpected_pass.test new file mode 100644 index 0000000000000..eede12ab0b014 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_unexpected_pass.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// XFAIL: * +// RUN: python3 %S/test_helper.py pass | FileCheck --check-prefix=CHECK-PASS %s +// CHECK-PASS: Test passed: result = 42 + +// This test is expected to fail (XFAIL) but actually passes. +// It should appear in "Unexpectedly Passed Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_unsupported.test b/unified-runtime/test/adapters/ci-validation/test_unsupported.test new file mode 100644 index 0000000000000..0a02802fef2dd --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_unsupported.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// UNSUPPORTED: linux, windows +// RUN: python3 %S/test_helper.py pass | FileCheck --check-prefix=CHECK-PASS %s +// CHECK-PASS: Test passed: result = 42 + +// This test is unsupported on all common platforms. +// It should appear in "Skipped" or "Unsupported Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_xfail.test b/unified-runtime/test/adapters/ci-validation/test_xfail.test new file mode 100644 index 0000000000000..0f12645c8b84e --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_xfail.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// XFAIL: * +// RUN: python3 %S/test_helper.py fail | FileCheck --check-prefix=CHECK-FAIL %s +// CHECK-FAIL: Test failed: expected 42, got 41 + +// This test is expected to fail (XFAIL). +// It should appear in "Expected Failures" category in CI logs. diff --git a/unified-runtime/test/adapters/cuda/CMakeLists.txt b/unified-runtime/test/adapters/cuda/CMakeLists.txt index 39c05d3a353c7..69d8cd0a1d01f 100644 --- a/unified-runtime/test/adapters/cuda/CMakeLists.txt +++ b/unified-runtime/test/adapters/cuda/CMakeLists.txt @@ -24,6 +24,7 @@ add_conformance_devices_test(adapter-cuda kernel_tests.cpp memory_tests.cpp event_tests.cpp + ci_validation_tests.cpp #FIXME: make this cleaner ${CMAKE_CURRENT_SOURCE_DIR}/../../../source/adapters/cuda/queue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../../source/adapters/cuda/common.cpp diff --git a/unified-runtime/test/adapters/cuda/ci_validation_tests.cpp b/unified-runtime/test/adapters/cuda/ci_validation_tests.cpp new file mode 100644 index 0000000000000..65e7a176ab71b --- /dev/null +++ b/unified-runtime/test/adapters/cuda/ci_validation_tests.cpp @@ -0,0 +1,39 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// CI Validation Tests +// These tests intentionally generate different test outcomes +// to validate CI logging and categorization + +#include + +// Test that always passes +TEST(CIValidation, test_pass) { EXPECT_EQ(42, 42); } + +// Test that always fails +TEST(CIValidation, test_fail) { + EXPECT_EQ(42, 41) << "Test failed: expected 42, got 41"; +} + +// Test expected to fail (XFAIL equivalent in Google Test) +// Disabled tests are skipped, which is close to XFAIL behavior +TEST(CIValidation, DISABLED_test_xfail) { + EXPECT_EQ(42, 41) << "Test failed: expected 42, got 41"; +} + +// Test expected to fail but actually passes (XPASS) +// This will show as disabled/skipped, but if someone runs it manually it passes +TEST(CIValidation, DISABLED_test_unexpected_pass) { EXPECT_EQ(42, 42); } + +// Test unsupported on common platforms +TEST(CIValidation, DISABLED_test_unsupported) { EXPECT_EQ(42, 42); } + +// Test that times out (infinite loop) +TEST(CIValidation, test_timeout) { + while (true) { + // Infinite loop to trigger timeout + } +} diff --git a/unified-runtime/test/conformance/CMakeLists.txt b/unified-runtime/test/conformance/CMakeLists.txt index 7d215f1b5ee14..23a0f1422daac 100644 --- a/unified-runtime/test/conformance/CMakeLists.txt +++ b/unified-runtime/test/conformance/CMakeLists.txt @@ -42,8 +42,10 @@ foreach(adapter ${UR_ADAPTERS_LIST}) if(NOT "${adapter}" STREQUAL "mock") if(UR_STANDALONE_BUILD) add_custom_target(check-unified-runtime-conformance-${adapter} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" - -v -Dselector=${adapter}:* + COMMAND ${CMAKE_COMMAND} -E env "LIT_OPTS=$ENV{LIT_OPTS}" + "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped + -Dselector=${adapter}:* DEPENDS deps_check-unified-runtime-conformance ) else() diff --git a/unified-runtime/third_party/requirements_testing.txt b/unified-runtime/third_party/requirements_testing.txt index c41dbabde3fe3..2ac825e67cfbd 100644 --- a/unified-runtime/third_party/requirements_testing.txt +++ b/unified-runtime/third_party/requirements_testing.txt @@ -4,3 +4,6 @@ lit==18.1.8 # For timeouts in lit psutil==7.0.0 + +# For secure XML parsing in test result processing +defusedxml==0.7.1