Skip to content

Commit 0ede2b8

Browse files
asamal4asamal4
authored
Merge pull request lightspeed-core#170 from VladimirKadlec/e2etests
feat: add e2e test with LSC api
2 parents 07629b6 + ecddcb3 commit 0ede2b8

7 files changed

Lines changed: 616 additions & 0 deletions

File tree

‎.github/workflows/e2e_tests.yaml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
# E2E integration tests with Lightspeed Core
2+
name: E2E Lightspeed Evaluation Integration Tests
3+
4+
on: [push, pull_request]
5+
6+
jobs:
7+
##########
8+
e2e_tests:
9+
runs-on: ubuntu-latest
10+
11+
#name: "Lightspeed-stack setup"
12+
13+
strategy:
14+
# For local testing use matrix with just one variant, "act" doesn't separate runs
15+
matrix:
16+
mode: ["query", "streaming"]
17+
eval-data: ["tests/integration/test_evaluation_data.yaml"]
18+
lsc_image_path: ["quay.io/lightspeed-core/lightspeed-stack:latest"]
19+
20+
name: "E2E Lightspeed Evaluation Test, mode: ${{ matrix.mode }}"
21+
22+
env:
23+
LSC_IMAGE_NAME: "lightspeed-stack-test-mode-${{ matrix.mode }}"
24+
25+
steps:
26+
# Stolen from lightspeed-stack
27+
- uses: actions/checkout@v4
28+
with:
29+
# On PR_TARGET → the fork (or same repo) that opened the PR.
30+
# On push → falls back to the current repository.
31+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
32+
33+
# On PR_TARGET → the PR head *commit* (reproducible).
34+
# On push → the pushed commit that triggered the workflow.
35+
ref: ${{ github.event.pull_request.head.ref || github.sha }}
36+
37+
# Don’t keep credentials when running untrusted PR code under PR_TARGET.
38+
persist-credentials: ${{ github.event_name != 'pull_request_target' }}
39+
40+
- name: Verify actual git checkout result
41+
run: |
42+
echo "=== Git Status After Checkout ==="
43+
echo "Remote URLs:"
44+
git remote -v
45+
echo ""
46+
echo "Current branch: $(git branch --show-current 2>/dev/null || echo 'detached HEAD')"
47+
echo "Current commit: $(git rev-parse HEAD)"
48+
echo "Current commit message: $(git log -1 --oneline)"
49+
echo ""
50+
echo "=== Recent commits ==="
51+
git log --oneline -5
52+
53+
# Run LSC
54+
# Can't be in onetime separate job -- networking is not shared between jobs
55+
- name: Run Lightspeed Stack (LSC)
56+
env:
57+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
58+
run: |
59+
echo "==========Running Lightspeed Core======="
60+
docker run \
61+
--name $LSC_IMAGE_NAME \
62+
-p 8080:8080 \
63+
-v $(pwd)/tests/integration/lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:Z \
64+
-v $(pwd)/tests/integration/run.yaml:/app-root/run.yaml:Z \
65+
-e OPENAI_API_KEY="${OPENAI_API_KEY}" \
66+
--detach \
67+
${{ matrix.lsc_image_path }}
68+
echo "==========Running Lightspeed Core Done======="
69+
70+
- name: Show logs from the LSC
71+
run: |
72+
sleep 2
73+
docker container ls -a
74+
docker logs $LSC_IMAGE_NAME
75+
76+
# Wait for LSC
77+
- name: Wait for the LSC
78+
run: |
79+
echo "Waiting for service on port 8080..."
80+
for i in {1..30}; do
81+
if curl --output /dev/null --fail http://localhost:8080/v1/models ; then
82+
echo "Service is up!"
83+
exit 0
84+
fi
85+
docker logs -n 10 $LSC_IMAGE_NAME
86+
echo "Still waiting..."
87+
sleep 2
88+
done
89+
90+
echo "Service did not start in time"
91+
exit 1
92+
93+
# Query mode
94+
- name: Set query mode
95+
if: matrix.mode == 'query'
96+
run: |
97+
echo "CONFIG=./tests/integration/system-config-query.yaml" >> $GITHUB_ENV
98+
99+
- name: Set streaming mode
100+
if: matrix.mode == 'streaming'
101+
run: |
102+
echo "CONFIG=./tests/integration/system-config-streaming.yaml" >> $GITHUB_ENV
103+
104+
# Dependencies
105+
- name: Install dependencies for Lightspeed Evaluation
106+
env:
107+
TERM: xterm-256color
108+
FORCE_COLOR: 1
109+
run: |
110+
echo "Installing e2e tests dependencies"
111+
pip install --break-system-packages uv
112+
uv sync
113+
114+
# Run the tests
115+
- name: Run the tests
116+
env:
117+
TERM: xterm-256color
118+
FORCE_COLOR: 1
119+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
120+
run: |
121+
echo "============================="
122+
echo "Running..."
123+
echo " config: ${CONFIG}"
124+
echo " LSC image: ${{ matrix.lsc_image_path }}"
125+
echo "============================="
126+
uv run lightspeed-eval --system-config "${CONFIG}" --eval-data "${{ matrix.eval-data }}"
127+
128+
# Check the result
129+
- name: Check test result
130+
run: |
131+
OUT_FILES=( eval_output/evaluation_*_summary.json )
132+
if [ ${#OUT_FILES[@]} != 1 ] ; then
133+
echo "Multiple output files: " eval_output/evaluation_*_summary.json
134+
exit 1
135+
fi
136+
OUT_FILE=${OUT_FILES[0]}
137+
PASS=$( jq .summary_stats.overall.PASS $OUT_FILE )
138+
EXPECTED="1"
139+
if [ ${PASS} != ${EXPECTED} ] ; then
140+
echo "============"
141+
echo "Wrong PASS number in ${OUT_FILE}: got ${PASS}, expected ${EXPECTED}"
142+
echo "============"
143+
exit 1
144+
fi
145+
146+
147+
# Cleanup
148+
- name: Stop the LSC if in local devel
149+
if: ${{ always() && env.ACT }}
150+
run: |
151+
echo "Stopping LSC container $LSC_IMAGE_NAME"
152+
echo "++++++++++++++++++++++"
153+
docker stop $LSC_IMAGE_NAME || true
154+
docker rm $LSC_IMAGE_NAME || true

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,6 @@ eval_output*/
189189
wip*/
190190

191191
.history/
192+
193+
# Used in e2e tests local testing
194+
.secrets

‎tests/integration/lightspeed-stack.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Lightspeed Core Service (LCS)
2+
service:
3+
host: 0.0.0.0
4+
port: 8080
5+
auth_enabled: false
6+
workers: 1
7+
color_log: true
8+
access_log: true
9+
llama_stack:
10+
# Uses a remote llama-stack service
11+
# The instance would have already been started with a llama-stack-run.yaml file
12+
# use_as_library_client: false
13+
14+
# Alternative for "as library use"
15+
use_as_library_client: true
16+
library_client_config_path: /app-root/run.yaml
17+
url: http://localhost:8321
18+
api_key: xyzzy
19+
user_data_collection:
20+
feedback_enabled: true
21+
feedback_storage: "/tmp/data/feedback"
22+
transcripts_enabled: true
23+
transcripts_storage: "/tmp/data/transcripts"
24+
25+
authentication:
26+
module: "noop"

‎tests/integration/run.yaml‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
version: 2
2+
3+
apis:
4+
- agents
5+
- batches
6+
- datasetio
7+
- eval
8+
- files
9+
- inference
10+
- safety
11+
- scoring
12+
- tool_runtime
13+
- vector_io
14+
15+
benchmarks: []
16+
datasets: []
17+
image_name: starter
18+
# external_providers_dir: /opt/app-root/src/.llama/providers.d
19+
20+
providers:
21+
inference:
22+
- provider_id: openai # This ID is a reference to 'providers.inference'
23+
provider_type: remote::openai
24+
config:
25+
api_key: ${env.OPENAI_API_KEY}
26+
allowed_models: ["${env.E2E_OPENAI_MODEL:=gpt-4o-mini}"]
27+
- config: {}
28+
provider_id: sentence-transformers
29+
provider_type: inline::sentence-transformers
30+
files:
31+
- config:
32+
metadata_store:
33+
table_name: files_metadata
34+
backend: sql_default
35+
storage_dir: ~/.llama/storage/files
36+
provider_id: meta-reference-files
37+
provider_type: inline::localfs
38+
safety:
39+
- config:
40+
excluded_categories: []
41+
provider_id: llama-guard
42+
provider_type: inline::llama-guard
43+
scoring:
44+
- provider_id: basic
45+
provider_type: inline::basic
46+
config: {}
47+
- provider_id: llm-as-judge
48+
provider_type: inline::llm-as-judge
49+
config: {}
50+
- provider_id: braintrust
51+
provider_type: inline::braintrust
52+
config:
53+
openai_api_key: '********'
54+
tool_runtime:
55+
- config: {} # Enable the RAG tool
56+
provider_id: rag-runtime
57+
provider_type: inline::rag-runtime
58+
vector_io:
59+
- config: # Define the storage backend for RAG
60+
persistence:
61+
namespace: vector_io::faiss
62+
backend: kv_default
63+
provider_id: faiss
64+
provider_type: inline::faiss
65+
agents:
66+
- config:
67+
persistence:
68+
agent_state:
69+
namespace: agents_state
70+
backend: kv_default
71+
responses:
72+
table_name: agents_responses
73+
backend: sql_default
74+
provider_id: meta-reference
75+
provider_type: inline::meta-reference
76+
batches:
77+
- config:
78+
kvstore:
79+
namespace: batches_store
80+
backend: kv_default
81+
provider_id: reference
82+
provider_type: inline::reference
83+
datasetio:
84+
- config:
85+
kvstore:
86+
namespace: huggingface_datasetio
87+
backend: kv_default
88+
provider_id: huggingface
89+
provider_type: remote::huggingface
90+
- config:
91+
kvstore:
92+
namespace: localfs_datasetio
93+
backend: kv_default
94+
provider_id: localfs
95+
provider_type: inline::localfs
96+
eval:
97+
- config:
98+
kvstore:
99+
namespace: eval_store
100+
backend: kv_default
101+
provider_id: meta-reference
102+
provider_type: inline::meta-reference
103+
scoring_fns: []
104+
server:
105+
port: 8321
106+
storage:
107+
backends:
108+
kv_default: # Define the storage backend type for RAG, in this case registry and RAG are unified i.e. information on registered resources (e.g. models, vector_stores) are saved together with the RAG chunks
109+
type: kv_sqlite
110+
db_path: ${env.KV_STORE_PATH:=~/.llama/storage/rag/kv_store.db}
111+
sql_default:
112+
type: sql_sqlite
113+
db_path: ${env.SQL_STORE_PATH:=~/.llama/storage/sql_store.db}
114+
stores:
115+
metadata:
116+
namespace: registry
117+
backend: kv_default
118+
inference:
119+
table_name: inference_store
120+
backend: sql_default
121+
max_write_queue_size: 10000
122+
num_writers: 4
123+
conversations:
124+
table_name: openai_conversations
125+
backend: sql_default
126+
prompts:
127+
namespace: prompts
128+
backend: kv_default
129+
registered_resources:
130+
models: []
131+
shields:
132+
- shield_id: llama-guard
133+
provider_id: llama-guard
134+
provider_shield_id: openai/gpt-4o-mini
135+
vector_stores: []
136+
datasets: []
137+
scoring_fns: []
138+
benchmarks: []
139+
tool_groups:
140+
- toolgroup_id: builtin::rag # Register the RAG tool
141+
provider_id: rag-runtime
142+
vector_stores:
143+
default_provider_id: faiss
144+
default_embedding_model: # Define the default embedding model for RAG
145+
provider_id: sentence-transformers
146+
model_id: nomic-ai/nomic-embed-text-v1.5
147+
safety:
148+
default_shield_id: llama-guard

0 commit comments

Comments
 (0)