RHEL AI E2E Tests #196
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/e2e_tests_rhelai.yaml | |
| name: RHEL AI E2E Tests | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Runs once a day at midnight UTC | |
| workflow_dispatch: | |
| jobs: | |
| e2e_tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| environment: ["rhelai"] | |
| # Expected default LLM for Behave (matches tests/e2e/configs/run-rhelai.yaml). | |
| # | environment | model_id (repo var) | provider_id | | |
| # |-------------|---------------------|-------------| | |
| # | rhelai | RHEL_AI_MODEL | vllm | | |
| include: | |
| - environment: rhelai | |
| e2e_default_provider: vllm | |
| e2e_default_model: ${{ vars.RHEL_AI_MODEL }} | |
| env: | |
| RHEL_AI_URL: ${{ secrets.RHEL_AI_URL }} | |
| RHEL_AI_PORT: ${{ secrets.RHEL_AI_PORT }} | |
| RHEL_AI_API_KEY: ${{ secrets.RHEL_AI_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| RHEL_AI_MODEL: ${{ vars.RHEL_AI_MODEL }} | |
| FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }} | |
| E2E_DEFAULT_MODEL_OVERRIDE: ${{ matrix.e2e_default_model }} | |
| E2E_DEFAULT_PROVIDER_OVERRIDE: ${{ matrix.e2e_default_provider }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # On PR_TARGET → the fork (or same repo) that opened the PR. | |
| # On push → falls back to the current repository. | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| # On PR_TARGET → the PR head *commit* (reproducible). | |
| # On push → the pushed commit that triggered the workflow. | |
| ref: ${{ github.event.pull_request.head.ref || github.sha }} | |
| # Don’t keep credentials when running untrusted PR code under PR_TARGET. | |
| persist-credentials: ${{ github.event_name != 'pull_request_target' }} | |
| - name: Verify actual git checkout result | |
| run: | | |
| echo "=== Git Status After Checkout ===" | |
| echo "Remote URLs:" | |
| git remote -v | |
| echo "" | |
| echo "Current branch: $(git branch --show-current 2>/dev/null || echo 'detached HEAD')" | |
| echo "Current commit: $(git rev-parse HEAD)" | |
| echo "Current commit message: $(git log -1 --oneline)" | |
| echo "" | |
| echo "=== Recent commits (should show setup-metrics commits) ===" | |
| git log --oneline -5 | |
| - uses: 1arp/create-a-file-action@0.4.5 | |
| with: | |
| path: '.' | |
| isAbsolutePath: false | |
| file: 'lightspeed-stack.yaml' | |
| content: | | |
| name: Lightspeed Core Service (LCS) | |
| service: | |
| host: 0.0.0.0 | |
| port: 8080 | |
| auth_enabled: false | |
| workers: 1 | |
| color_log: true | |
| access_log: true | |
| llama_stack: | |
| # Uses a remote llama-stack service | |
| # The instance would have already been started with a llama-stack-run.yaml file | |
| use_as_library_client: false | |
| # Alternative for "as library use" | |
| # use_as_library_client: true | |
| # library_client_config_path: <path-to-llama-stack-run.yaml-file> | |
| url: http://llama-stack:8321 | |
| api_key: xyzzy | |
| user_data_collection: | |
| feedback_enabled: true | |
| feedback_storage: "/tmp/data/feedback" | |
| transcripts_enabled: true | |
| transcripts_storage: "/tmp/data/transcripts" | |
| authentication: | |
| module: "noop" | |
| - name: Select and configure run.yaml | |
| env: | |
| CONFIG_ENVIRONMENT: ${{ matrix.environment || 'rhelai' }} | |
| run: | | |
| CONFIGS_DIR="tests/e2e/configs" | |
| ENVIRONMENT="$CONFIG_ENVIRONMENT" | |
| echo "Looking for configurations in $CONFIGS_DIR/" | |
| # List available configurations | |
| if [ -d "$CONFIGS_DIR" ]; then | |
| echo "Available configurations:" | |
| ls -la "$CONFIGS_DIR"/*.yaml 2>/dev/null || echo "No YAML files found in $CONFIGS_DIR/" | |
| else | |
| echo "Configs directory '$CONFIGS_DIR' not found!" | |
| exit 1 | |
| fi | |
| # Determine which config file to use | |
| CONFIG_FILE="$CONFIGS_DIR/run-$ENVIRONMENT.yaml" | |
| echo "Looking for: $CONFIG_FILE" | |
| if [ -f "$CONFIG_FILE" ]; then | |
| echo "Found config for environment: $ENVIRONMENT" | |
| cp "$CONFIG_FILE" run.yaml | |
| else | |
| echo "Configuration file not found: $CONFIG_FILE" | |
| echo "Available files in $CONFIGS_DIR:" | |
| ls -la "$CONFIGS_DIR/" | |
| exit 1 | |
| fi | |
| # Update paths for container environment (relative -> absolute) | |
| sed -i 's|db_path: \.llama/distributions|db_path: /app-root/.llama/distributions|g' run.yaml | |
| sed -i 's|db_path: tmp/|db_path: /app-root/.llama/distributions/|g' run.yaml | |
| echo "Successfully configured for environment: $ENVIRONMENT" | |
| echo "Using configuration: $(basename "$CONFIG_FILE")" | |
| - name: Test RHEL_AI connectivity | |
| run: | | |
| echo $RHEL_AI_MODEL | |
| echo "GET ${RHEL_AI_URL}:${RHEL_AI_PORT}/v1/models" | |
| RESP=$(curl -sS -w "\n%{http_code}" "${RHEL_AI_URL}:${RHEL_AI_PORT}/v1/models" \ | |
| -H "Authorization: Bearer ${RHEL_AI_API_KEY}" || printf '\n000') | |
| HTTP_CODE=$(echo "$RESP" | tail -n1) | |
| BODY=$(echo "$RESP" | sed '$d') | |
| echo "HTTP $HTTP_CODE" | |
| echo "Response body:" | |
| echo "$BODY" | |
| [ "$HTTP_CODE" = "200" ] | |
| - name: Docker Login for quay access | |
| env: | |
| QUAY_ROBOT_USERNAME: ${{ secrets.QUAY_DOWNSTREAM_USERNAME }} | |
| QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_DOWNSTREAM_TOKEN }} | |
| run: | | |
| echo $QUAY_ROBOT_TOKEN | docker login quay.io -u=$QUAY_ROBOT_USERNAME --password-stdin | |
| - name: Run service manually | |
| run: | | |
| docker compose version | |
| docker compose up -d | |
| # Check for errors and show logs if any services failed | |
| if docker compose ps | grep -E 'Exit|exited|stopped'; then | |
| echo "Some services failed to start - showing logs:" | |
| docker compose logs | |
| exit 1 | |
| else | |
| echo "All services started successfully" | |
| fi | |
| - name: Wait for services | |
| run: | | |
| echo "Waiting for services to be healthy..." | |
| sleep 20 # adjust depending on boot time | |
| - name: Quick connectivity test | |
| env: | |
| EXPECTED_PROVIDER: ${{ matrix.e2e_default_provider }} | |
| EXPECTED_MODEL: ${{ matrix.e2e_default_model }} | |
| run: | | |
| set -euo pipefail | |
| show_logs() { | |
| docker compose logs | |
| } | |
| echo "Testing /v1/models and verifying expected LLM is registered..." | |
| echo "Expected: provider_id=${EXPECTED_PROVIDER} provider_resource_id=${EXPECTED_MODEL}" | |
| CURL_EC=0 | |
| MODEL_JSON=$(curl -sS "http://localhost:8080/v1/models?model_type=llm") || CURL_EC=$? | |
| echo "GET /v1/models?model_type=llm response:" | |
| echo "$MODEL_JSON" | |
| if [ "$CURL_EC" -ne 0 ]; then | |
| echo "❌ GET /v1/models?model_type=llm failed (curl exit $CURL_EC)" | |
| show_logs | |
| exit 1 | |
| fi | |
| if ! echo "$MODEL_JSON" | jq -e --arg p "$EXPECTED_PROVIDER" --arg m "$EXPECTED_MODEL" \ | |
| '.models | any(.provider_id == $p and .provider_resource_id == $m)' >/dev/null | |
| then | |
| echo "❌ Expected LLM not listed by /v1/models (provider_id + provider_resource_id must match)" | |
| echo "Registered LLM entries (provider_id, provider_resource_id, api_model_type):" | |
| echo "$MODEL_JSON" | jq -r '.models[] | "\(.provider_id)\t\(.provider_resource_id)\t\(.api_model_type)"' || true | |
| show_logs | |
| exit 1 | |
| fi | |
| echo "✅ Connectivity OK; expected model is registered (${EXPECTED_PROVIDER} / ${EXPECTED_MODEL})" | |
| - name: Run e2e tests | |
| run: | | |
| echo "Installing test dependencies..." | |
| pip install uv | |
| uv sync | |
| echo "Running comprehensive e2e test suite..." | |
| make test-e2e | |
| - name: Show logs on failure | |
| if: failure() | |
| run: | | |
| echo "=== Test failure logs ===" | |
| echo "=== llama-stack logs ===" | |
| docker compose logs llama-stack | |
| echo "" | |
| echo "=== lightspeed-stack logs ===" | |
| docker compose logs lightspeed-stack |