evaluation_data.yaml

# Lightspeed Evaluation Framework - Sample/Mock Data

- conversation_group_id: conv_group_1
  description: conversation group description
  tag: basic  # Tag for grouping eval conversations (default: "eval")

  conversation_metrics: []
  conversation_metrics_metadata: {}
  turns:
    - turn_id: turn_id1
      query: User query
      attachments: null
      response: API response
      tool_calls: null
      contexts:
        - Context 1
        - Context 2
      expected_response: Expected Response
      expected_tool_calls: null
      turn_metrics:
      - ragas:faithfulness
      - ragas:response_relevancy
      - ragas:context_utilization
      turn_metrics_metadata:
        ragas:faithfulness:
          threshold: 0.99
      verify_script: null
  setup_script: null
  cleanup_script: null

- conversation_group_id: conv_group_2
  description: conversation group description
  conversation_metrics: []
  conversation_metrics_metadata: {}
  turns:
    - turn_id: turn_id1
      query: User Query
      response: Sample response based on context  # Required when API disabled
      contexts:
        - Context 1
      expected_response: Expected Response
      turn_metrics:
        - ragas:context_recall
        - ragas:context_relevance
        - ragas:context_precision

- conversation_group_id: conv_group_3
  description: conversation group description
  conversation_metrics:
    - deepeval:conversation_completeness
    - deepeval:conversation_relevancy
  turns:
    - turn_id: turn_id1  # Skip turn-level evaluation
      query: User Query 1
      response: Response to query 1  # Required for conversation metrics
      turn_metrics: []
    - turn_id: turn_id2  # Default evaluation
      query: User Query 2
      response: Response to query 2  # Required for conversation metrics

- conversation_group_id: tool_eval_example
  description: Example of tool call evaluation with direct and regex matching
  tag: tools

  turns:
    - turn_id: turn_id1
      query: Get all pods in the default namespace
      tool_calls:
        -
          - tool_name: oc_get
            arguments:
              kind: pod
              namespace: default
      expected_tool_calls:
        -
          - tool_name: oc_get
            arguments:
              kind: pod
              namespace: default
      turn_metrics:
        - custom:tool_eval

    - turn_id: turn_id2
      query: Get all pods in the openshift-lightspeed namespace 
      tool_calls:
        -
          - tool_name: oc_get
            arguments:
              kind: pod
              namespace: openshift-lightspeed
      expected_tool_calls:
        -
          - tool_name: oc_get
            arguments:
              kind: pod
              namespace: openshift-light.*  # Regex pattern for flexible matching
      turn_metrics:
        - custom:tool_eval

- conversation_group_id: tool_eval_with_result
  description: Example of tool call evaluation including result validation
  tag: tools

  turns:
    - turn_id: turn_id1
      query: Get the status of pod nginx-123 in the default namespace
      tool_calls:
        -
          - tool_name: oc_get
            arguments:
              kind: pod
              name: nginx-123
              namespace: default
            result: "pod/nginx-123 Running 1/1"
      expected_tool_calls:
        -
          - tool_name: oc_get
            arguments:
              kind: pod
              name: nginx-.*
              namespace: default
            result: ".*Running.*"  # Regex pattern to verify pod is running
      turn_metrics:
        - custom:tool_eval

    - turn_id: turn_id2
      query: Create a new namespace called test-ns
      tool_calls:
        -
          - tool_name: oc_create
            arguments:
              kind: namespace
              name: test-ns
            result: "namespace/test-ns created"
      expected_tool_calls:
        -
          - tool_name: oc_create
            arguments:
              kind: namespace
              name: test-ns
            result: ".*created"  # Verify creation succeeded
      turn_metrics:
        - custom:tool_eval

- conversation_group_id: script_eval_example
  description: Example of script-based evaluation for namespace testing
  tag: infrastructure
  setup_script: sample_scripts/setup.sh
  cleanup_script: sample_scripts/cleanup.sh

  turns:
    - turn_id: turn_id1
      query: Create a namespace called ols-test-ns
      turn_metrics:
        - script:action_eval
      verify_script: sample_scripts/verify.sh

# NLP Metrics Examples - Text comparison without LLM
# These metrics compare response vs expected_response using traditional NLP techniques
#
# REQUIRED: Install optional dependencies first:
#   pip install 'lightspeed-evaluation[nlp-metrics]'
#   OR: pip install sacrebleu rouge-score rapidfuzz

- conversation_group_id: nlp_metrics_basic
  description: Basic NLP metrics - BLEU, ROUGE, and Semantic Similarity with defaults

  turns:
    - turn_id: turn_id1
      query: What is the capital of France?
      response: The capital of France is Paris, a major European city.
      expected_response: Paris is the capital of France.
      turn_metrics:
        - nlp:bleu              # N-gram overlap (0-1, higher = more similar)
        - nlp:rouge             # Recall-oriented overlap (default: rougeL, fmeasure)
        # - nlp:semantic_similarity_distance  # String distance (NOT recommended for LLM outputs)

- conversation_group_id: nlp_metrics_custom_config
  description: NLP metrics with custom configuration options

  turns:
    - turn_id: turn_id1
      query: Explain how to deploy a pod in Kubernetes
      response: Use kubectl apply -f pod.yaml to deploy a pod to your Kubernetes cluster.
      expected_response: To deploy a pod in Kubernetes, use the kubectl apply command with your pod manifest file.
      turn_metrics:
        - nlp:bleu
        - nlp:rouge
        - nlp:semantic_similarity_distance  # Caution: measures character distance, not meaning
      turn_metrics_metadata:
        nlp:bleu:
          threshold: 0.3  # Custom threshold for BLEU
          max_ngram: 2    # Use bigrams instead of default 4-grams (options: 1-4)
        nlp:rouge:
          threshold: 0.3
          rouge_type: rouge1  # Options: rouge1, rouge2, rougeL, rougeLsum
        nlp:semantic_similarity_distance:
          threshold: 0.6
          distance_measure: jaro_winkler  # Options: levenshtein, hamming, jaro, jaro_winkler