feat: add tracing (#85)

Author: mathislucka

pyproject.toml

@@ -102,7 +102,7 @@ convention = "pep257"
 strict = true
 
 [[tool.mypy.overrides]]
-module = ["haystack_integrations.*"]
+module = ["haystack_integrations.*", "langfuse.*"]
 ignore_missing_imports = true
 
 

src/deepset_mcp/benchmark/runner/agent_benchmark_runner.py

@@ -16,6 +16,7 @@
 )
 from deepset_mcp.benchmark.runner.models import AgentConfig, TestCaseConfig
 from deepset_mcp.benchmark.runner.teardown_actions import teardown_test_case_async
+from deepset_mcp.benchmark.runner.tracing import enable_tracing
 
 logger = logging.getLogger(__name__)
 
@@ -38,6 +39,14 @@ def __init__(
         self.agent_config = agent_config
         self.benchmark_config = benchmark_config
 
+        try:
+            secret_key = self.benchmark_config.get_env_var("LANGFUSE_SECRET_KEY")
+            public_key = self.benchmark_config.get_env_var("LANGFUSE_PUBLIC_KEY")
+            logger.info("Langfuse environment variables detected. Enabling tracing with Langfuse.")
+            enable_tracing(secret_key=secret_key, public_key=public_key, name="deepset-mcp")
+        except KeyError:
+            pass
+
         agent, commit_hash = load_agent(config=agent_config, benchmark_config=benchmark_config)
 
         self.agent = agent

src/deepset_mcp/benchmark/runner/cli_agent.py

@@ -24,7 +24,7 @@ def load_env_file(env_file: str | None) -> None:
         default_env_path = Path(__file__).parent / ".env"
         if default_env_path.exists():
             load_dotenv()
-            typer.secho("Loaded default .env file", fg=typer.colors.BLUE)
+            typer.secho("Loaded default .env file.", fg=typer.colors.BLUE)
 
 
 agent_app = typer.Typer(help="Commands for running agents against test cases.")

src/deepset_mcp/benchmark/runner/config.py

@@ -37,7 +37,7 @@ def validate_not_empty(cls, v: str) -> str:
     def model_post_init(self, __context: Any) -> None:
         """After initialization, collect all available env vars."""
         # Collect all env vars that might be useful (excluding system ones)
-        ignore_prefixes = ("PATH", "HOME", "USER", "SHELL", "TERM", "PWD", "LANG", "LC_")
+        ignore_prefixes = ("PATH", "HOME", "USER", "SHELL", "TERM", "PWD", "LC_")
 
         for key, value in os.environ.items():
             # Skip system variables and already captured ones

src/deepset_mcp/benchmark/runner/tracing.py

@@ -0,0 +1,21 @@
+from haystack.tracing.tracer import enable_tracing as haystack_enable_tracing, tracer
+from haystack_integrations.tracing.langfuse import LangfuseTracer
+from langfuse import Langfuse
+
+
+def enable_tracing(
+    secret_key: str,
+    public_key: str,
+    name: str,
+) -> None:
+    """Enables tracing with langfuse."""
+    resolved_langfuse_client_kwargs = {
+        "secret_key": secret_key,
+        "public_key": public_key,
+    }
+    tracer.is_content_tracing_enabled = True
+    langfuse_tracer = LangfuseTracer(
+        tracer=Langfuse(**resolved_langfuse_client_kwargs),
+        name=name,
+    )
+    haystack_enable_tracing(langfuse_tracer)