Skip to content

Commit 39851a5

Browse files
adityamehraadityamehra
committed
test(llamaindex): enable all tests to run without OpenAI API key
Replace OpenAI LLM/Embedding with MockLLM/MockEmbedding so tests run in CI without credentials. Use session-scoped instrumentation fixture in conftest.py to avoid event-stream-consumed errors from incomplete uninstrument. Remove module-level skip markers from embedding, RAG, and workflow tests. Only test_with_openai remains skipped (needs live API key). 17 passed, 1 skipped locally. Made-with: Cursor
1 parent f23a847 commit 39851a5

7 files changed

Lines changed: 313 additions & 748 deletions

File tree

‎instrumentation-genai/opentelemetry-instrumentation-llamaindex/tests/conftest.py‎

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,91 @@
44

55
import pytest
66

7+
from opentelemetry.instrumentation.llamaindex import LlamaindexInstrumentor
8+
from opentelemetry.sdk.metrics import MeterProvider
9+
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
10+
from opentelemetry.sdk.trace import TracerProvider
11+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
12+
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
13+
InMemorySpanExporter,
14+
)
715
from opentelemetry.util.genai import handler as genai_handler
816

17+
_session_span_exporter = InMemorySpanExporter()
18+
_session_metric_reader = InMemoryMetricReader()
19+
920

1021
@pytest.fixture(autouse=True)
11-
def disable_deepeval():
12-
"""Disable deepeval evaluators to prevent real API calls in CI."""
13-
original_evals = os.environ.get("OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS")
22+
def environment():
23+
"""Reset env and handler singleton for each test."""
24+
original_evals = os.environ.get(
25+
"OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"
26+
)
27+
original_emitters = os.environ.get(
28+
"OTEL_INSTRUMENTATION_GENAI_EMITTERS"
29+
)
1430

1531
os.environ["OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"] = "none"
16-
setattr(genai_handler.get_telemetry_handler, "_default_handler", None)
32+
os.environ["OTEL_INSTRUMENTATION_GENAI_EMITTERS"] = "span_metric"
1733

1834
yield
1935

2036
if original_evals is None:
21-
os.environ.pop("OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS", None)
37+
os.environ.pop(
38+
"OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS", None
39+
)
40+
else:
41+
os.environ["OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"] = (
42+
original_evals
43+
)
44+
45+
if original_emitters is None:
46+
os.environ.pop("OTEL_INSTRUMENTATION_GENAI_EMITTERS", None)
2247
else:
23-
os.environ["OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"] = original_evals
48+
os.environ["OTEL_INSTRUMENTATION_GENAI_EMITTERS"] = (
49+
original_emitters
50+
)
2451

52+
53+
@pytest.fixture(scope="session", autouse=True)
54+
def _instrument_once():
55+
"""Instrument LlamaIndex once for the entire test session."""
56+
os.environ["OTEL_INSTRUMENTATION_GENAI_EVALS_EVALUATORS"] = "none"
57+
os.environ["OTEL_INSTRUMENTATION_GENAI_EMITTERS"] = "span_metric"
2558
setattr(genai_handler.get_telemetry_handler, "_default_handler", None)
59+
60+
tracer_provider = TracerProvider()
61+
tracer_provider.add_span_processor(
62+
SimpleSpanProcessor(_session_span_exporter)
63+
)
64+
65+
meter_provider = MeterProvider(
66+
metric_readers=[_session_metric_reader]
67+
)
68+
69+
instrumentor = LlamaindexInstrumentor()
70+
instrumentor._is_instrumented_by_opentelemetry = False
71+
instrumentor.instrument(
72+
tracer_provider=tracer_provider,
73+
meter_provider=meter_provider,
74+
)
75+
76+
yield instrumentor
77+
78+
79+
@pytest.fixture
80+
def span_exporter():
81+
"""Provide a cleared span exporter for each test."""
82+
_session_span_exporter.clear()
83+
yield _session_span_exporter
84+
85+
86+
@pytest.fixture
87+
def metric_reader():
88+
yield _session_metric_reader
89+
90+
91+
@pytest.fixture
92+
def instrument(_instrument_once):
93+
"""Marker fixture for tests that need instrumentation."""
94+
yield _instrument_once

‎instrumentation-genai/opentelemetry-instrumentation-llamaindex/tests/test_embedding_instrumentation.py‎

Lines changed: 17 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -1,159 +1,43 @@
11
"""Test embedding instrumentation for LlamaIndex."""
22

3-
import pytest
4-
5-
import os
6-
73
from llama_index.core import Settings
84
from llama_index.core.callbacks import CallbackManager
9-
from llama_index.embeddings.openai import OpenAIEmbedding
10-
11-
from opentelemetry import metrics, trace
12-
from opentelemetry.instrumentation.llamaindex import LlamaindexInstrumentor
13-
from opentelemetry.sdk.metrics import MeterProvider
14-
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
15-
from opentelemetry.sdk.trace import TracerProvider
16-
from opentelemetry.sdk.trace.export import (
17-
ConsoleSpanExporter,
18-
SimpleSpanProcessor,
19-
)
20-
21-
pytestmark = pytest.mark.skip(
22-
reason="Requires live OpenAI API key; needs VCR cassettes"
23-
)
24-
25-
# Global setup - shared across tests
26-
metric_reader = None
27-
instrumentor = None
28-
29-
30-
def setup_telemetry():
31-
"""Setup OpenTelemetry with span and metric exporters (once)."""
32-
global metric_reader, instrumentor
33-
34-
if metric_reader is not None:
35-
return metric_reader
36-
37-
# Enable metrics
38-
os.environ["OTEL_INSTRUMENTATION_GENAI_EMITTERS"] = "span_metric"
39-
40-
# Setup tracing
41-
trace.set_tracer_provider(TracerProvider())
42-
trace.get_tracer_provider().add_span_processor(
43-
SimpleSpanProcessor(ConsoleSpanExporter())
44-
)
45-
46-
# Setup metrics with InMemoryMetricReader
47-
metric_reader = InMemoryMetricReader()
48-
meter_provider = MeterProvider(metric_readers=[metric_reader])
49-
metrics.set_meter_provider(meter_provider)
50-
51-
# Enable instrumentation once
52-
instrumentor = LlamaindexInstrumentor()
53-
instrumentor.instrument(
54-
tracer_provider=trace.get_tracer_provider(),
55-
meter_provider=metrics.get_meter_provider(),
56-
)
57-
58-
return metric_reader
5+
from llama_index.core.embeddings import MockEmbedding
596

607

61-
def test_embedding_single_text():
62-
"""Test single text embedding instrumentation."""
63-
print("\nTest: Single Text Embedding")
64-
print("=" * 60)
65-
66-
metric_reader = setup_telemetry()
67-
68-
# Configure embedding model
69-
embed_model = OpenAIEmbedding(
70-
model="text-embedding-3-small",
71-
api_key=os.environ.get("OPENAI_API_KEY"),
72-
)
8+
def test_embedding_single_text(span_exporter, instrument):
9+
"""Test single text embedding produces spans."""
10+
embed_model = MockEmbedding(embed_dim=8)
7311
Settings.embed_model = embed_model
74-
75-
# Make sure callback manager is initialized
7612
if Settings.callback_manager is None:
7713
Settings.callback_manager = CallbackManager()
7814

79-
# Generate single embedding
80-
text = "LlamaIndex is a data framework for LLM applications"
81-
embedding = embed_model.get_text_embedding(text)
82-
83-
print(f"\nText: {text}")
84-
print(f"Embedding dimension: {len(embedding)}")
85-
print(f"First 5 values: {embedding[:5]}")
86-
87-
# Validate metrics
88-
print("\nMetrics:")
89-
metrics_data = metric_reader.get_metrics_data()
90-
for resource_metric in metrics_data.resource_metrics:
91-
for scope_metric in resource_metric.scope_metrics:
92-
for metric in scope_metric.metrics:
93-
print(f"\nMetric: {metric.name}")
94-
for data_point in metric.data.data_points:
95-
if hasattr(data_point, "bucket_counts"):
96-
# Histogram
97-
print(f" Count: {sum(data_point.bucket_counts)}")
98-
else:
99-
# Counter
100-
print(f" Value: {data_point.value}")
101-
102-
print("\nTest completed successfully")
15+
embedding = embed_model.get_text_embedding(
16+
"LlamaIndex is a data framework for LLM applications"
17+
)
10318

19+
assert len(embedding) == 8
10420

105-
def test_embedding_batch():
106-
"""Test batch embedding instrumentation."""
107-
print("\nTest: Batch Embeddings")
108-
print("=" * 60)
21+
spans = span_exporter.get_finished_spans()
22+
assert len(spans) >= 1
10923

110-
metric_reader = setup_telemetry()
11124

112-
# Configure embedding model
113-
embed_model = OpenAIEmbedding(
114-
model="text-embedding-3-small",
115-
api_key=os.environ.get("OPENAI_API_KEY"),
116-
)
25+
def test_embedding_batch(span_exporter, instrument):
26+
"""Test batch embedding produces spans."""
27+
embed_model = MockEmbedding(embed_dim=8)
11728
Settings.embed_model = embed_model
118-
119-
# Make sure callback manager is initialized
12029
if Settings.callback_manager is None:
12130
Settings.callback_manager = CallbackManager()
12231

123-
# Generate batch embeddings
12432
texts = [
12533
"Paris is the capital of France",
12634
"Berlin is the capital of Germany",
12735
"Rome is the capital of Italy",
12836
]
12937
embeddings = embed_model.get_text_embedding_batch(texts)
13038

131-
print(f"\nEmbedded {len(embeddings)} texts")
132-
print(f"Dimension: {len(embeddings[0])}")
133-
134-
# Validate metrics
135-
print("\nMetrics:")
136-
metrics_data = metric_reader.get_metrics_data()
137-
for resource_metric in metrics_data.resource_metrics:
138-
for scope_metric in resource_metric.scope_metrics:
139-
for metric in scope_metric.metrics:
140-
print(f"\nMetric: {metric.name}")
141-
for data_point in metric.data.data_points:
142-
if hasattr(data_point, "bucket_counts"):
143-
# Histogram
144-
print(f" Count: {sum(data_point.bucket_counts)}")
145-
else:
146-
# Counter
147-
print(f" Value: {data_point.value}")
148-
149-
print("\nTest completed successfully")
150-
151-
152-
if __name__ == "__main__":
153-
test_embedding_single_text()
154-
print("\n" + "=" * 60 + "\n")
155-
test_embedding_batch()
39+
assert len(embeddings) == 3
40+
assert all(len(e) == 8 for e in embeddings)
15641

157-
# Cleanup
158-
if instrumentor:
159-
instrumentor.uninstrument()
42+
spans = span_exporter.get_finished_spans()
43+
assert len(spans) >= 1

0 commit comments

Comments
 (0)