FIRE-570 | Qualifire tracing (#244)

* Add qualifire tracing

* Add missing files

* Codestyle

* Updategitignore

* Disable otel metrics and logging

* update poetry.lock

* poetry fixes and codestyle

* Move consts to separate file

* Add traceloop dependency

* handle lower python versions in code

* Delete historic tests, create tests for utils

* Delete historic tests, create tests for utils

* Fix tests

* Fix tests

* Disable telemetry and traceloop sync

Author: yuval-qf

.gitignore

@@ -34,6 +34,8 @@ Temporary Items
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
+.idea/
+
 # User-specific stuff
 .idea/**/workspace.xml
 .idea/**/tasks.xml

.pre-commit-config.yaml

@@ -2,14 +2,14 @@ default_stages: [commit, push]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
         exclude: LICENSE
 
   - repo: https://github.com/pycqa/flake8
-    rev: "3.9.2" # pick a git hash / tag to point to
+    rev: "7.2.0" # pick a git hash / tag to point to
     hooks:
       - id: flake8
 

poetry.lock

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "qualifire"
-version = "0.7.0"
+version = "0.8.0"
 description = "Qualifire Python SDK"
 readme = "README.md"
 authors = ["qualifire-dev <dror@qualifire.ai>"]
@@ -33,6 +33,7 @@ classifiers = [  #! Update me
 
 [tool.poetry.dependencies]
 python = "^3.8"
+traceloop-sdk = { version = "0.40.4" , markers = "python_version >= '3.10'" }
 
 [tool.poetry.dev-dependencies]
 bandit = "^1.7.1"

pyproject.toml

@@ -1,16 +1,11 @@
-"""Qualifire PYthon SDK"""
+"""Qualifire Python SDK"""
 
 import logging
-import os
 
-import pkg_resources
-
-from . import client, types
+from . import client, consts, tracer_init, types, utils
+from .tracer_init import init
 
 logger = logging.getLogger("qualifire")
-QUALIFIRE_API_KEY_ENV_VAR = "QUALIFIRE_API_KEY"
-QUALIFIRE_BASE_URL_ENV_VAR = "QUALIFIRE_BASE_URL"
-_DEFAULT_BASE_URL = "https://proxy.qualifire.ai/"
 
 
 from importlib import metadata as importlib_metadata
@@ -24,3 +19,10 @@ def get_version() -> str:
 
 
 version: str = get_version()
+
+__all__ = [
+    "client",
+    "types",
+    "init",
+    "version",
+]

qualifire/__init__.py

@@ -7,20 +7,21 @@
 import requests
 
 from .types import EvaluationRequest, EvaluationResponse, LLMMessage, SyntaxCheckArgs
+from .utils import get_api_key, get_base_url
 
 logger = logging.getLogger("qualifire")
 
 
 class Client:
     def __init__(
         self,
-        api_key: str,
-        base_url: str = "https://proxy.qualifire.ai",
-        version: str = None,
+        api_key: Optional[str],
+        base_url: Optional[str] = None,
+        version: Optional[str] = None,
         debug: bool = False,
     ) -> None:
-        self._base_url = base_url
-        self._api_key = api_key
+        self._base_url = base_url or get_base_url()
+        self._api_key = api_key or get_api_key()
         self._version = version
         self._debug = debug
 
@@ -47,8 +48,6 @@ def evaluate(
         :param input: The primary input for the evaluation.
         :param output: The primary output (e.g., LLM response) to evaluate.
         :param assertions: A list of custom assertions to check against the output.
-        :param consistency_check: Check for consistency between
-        input/output and context.
         :param dangerous_content_check: Check for dangerous content generation.
         :param grounding_check: Check if the output is grounded in the provided
                                 input/context.
@@ -60,8 +59,6 @@ def evaluate(
         :param messages: List of message objects representing conversation history.
         :param pii_check: Check for personally identifiable information.
         :param prompt_injections: Check for attempts at prompt injection.
-        :param responseFunctionCalls: Expected function calls in the response
-        (if applicable).
         :param sexual_content_check: Check for sexually explicit content.
         :param syntax_checks: Dictionary defining syntax checks (e.g., JSON, SQL).
 
@@ -80,7 +77,6 @@ def evaluate(
             input="Translate 'hello' to French and provide the result in JSON format.",
             output='{"translation": "bonjour"}',
             assertions=["The output must contain the key 'translation'"],
-            consistency_check=True,
             dangerous_content_check=True,
             grounding_check=True,
             hallucinations_check=True,
@@ -96,7 +92,6 @@ def evaluate(
             ],
             pii_check=True,
             prompt_injections=True,
-            # responseFunctionCalls="some_function_call", # Optional
             sexual_content_check=True,
             syntax_checks={
                 "json": SyntaxCheckArgs(args="strict") # Example syntax check
@@ -135,16 +130,15 @@ def evaluate(
         if response.status_code != 200:
             raise Exception(f"Qualifire API error: {response.text}")
 
-        jsonResponse = response.json()
-        return EvaluationResponse(**jsonResponse)
+        json_response = response.json()
+        return EvaluationResponse(**json_response)
 
     def invoke_evaluation(
         self,
         input: str,
         output: str,
         evaluation_id: str,
-    ):
-
+    ) -> EvaluationResponse:
         url = f"{self._base_url}/api/evaluation/invoke/"
 
         payload = {
@@ -163,5 +157,5 @@ def invoke_evaluation(
                 response.raise_for_status()
             raise Exception(f"Qualifire API error: {response.text}")
 
-        jsonResponse = response.json()
-        return EvaluationResponse(**jsonResponse)
+        json_response = response.json()
+        return EvaluationResponse(**json_response)

qualifire/client.py

@@ -0,0 +1,3 @@
+QUALIFIRE_API_KEY_ENV_VAR = "QUALIFIRE_API_KEY"
+QUALIFIRE_BASE_URL_ENV_VAR = "QUALIFIRE_BASE_URL"
+_DEFAULT_BASE_URL = "https://proxy.qualifire.ai/"

qualifire/consts.py

@@ -0,0 +1,55 @@
+from typing import Any, Callable, Optional, TypeVar
+
+import os
+import sys
+
+try:
+    from traceloop.sdk import Traceloop
+
+    traceloop_installed = True
+except ImportError:
+    traceloop_installed = False
+
+from .utils import get_api_key, get_base_url
+
+R = TypeVar("R")
+
+
+def __suppress_prints(
+    func: Callable[..., R],
+    *args: Any,
+    **kwargs: Any,
+) -> R:
+    original_stdout = sys.stdout
+    sys.stdout = open(os.devnull, "w")
+    try:
+        return func(*args, **kwargs)
+    finally:
+        sys.stdout.close()
+        sys.stdout = original_stdout
+
+
+def __configure_tracer(api_key: str) -> None:
+    # traceloop uses env for those configs
+    os.environ["TRACELOOP_METRICS_ENABLED"] = "false"
+    os.environ["TRACELOOP_LOGGING_ENABLED"] = "false"
+
+    __suppress_prints(
+        Traceloop.init,
+        app_name="qualifire-agent",
+        api_endpoint=f"{get_base_url()}/telemetry",  # /v1/traces is automatically added  # noqa: E501
+        headers={"X-Qualifire-API-Key": api_key},
+        telemetry_enabled=False,
+        traceloop_sync_enabled=False,
+    )
+
+
+def init(api_key: Optional[str] = None) -> None:
+    if not traceloop_installed:
+        if sys.version_info < (3, 10):
+            raise RuntimeError("qualifire.init requires Python 3.10 or higher")
+        else:
+            raise RuntimeError("Dependency error, please reinstall qualifire-sdk")
+
+    api_key = api_key or get_api_key()
+    __configure_tracer(api_key)

qualifire/tracer_init.py

@@ -0,0 +1,21 @@
+import os
+
+from .consts import (
+    _DEFAULT_BASE_URL,
+    QUALIFIRE_API_KEY_ENV_VAR,
+    QUALIFIRE_BASE_URL_ENV_VAR,
+)
+
+
+def get_api_key() -> str:
+    api_key = os.getenv(QUALIFIRE_API_KEY_ENV_VAR)
+    if not api_key:
+        raise ValueError("Qualifire API Key not found")
+    return api_key
+
+
+def get_base_url() -> str:
+    base_url = os.getenv(QUALIFIRE_BASE_URL_ENV_VAR)
+    if not base_url:
+        return _DEFAULT_BASE_URL
+    return base_url

qualifire/utils.py

@@ -12,3 +12,4 @@ requests==2.31.0 ; python_version >= "3.8" and python_version < "4.0"
 tqdm==4.66.1 ; python_version >= "3.8" and python_version < "4.0"
 urllib3==2.1.0 ; python_version >= "3.8" and python_version < "4.0"
 yarl==1.9.4 ; python_version >= "3.8" and python_version < "4.0"
+traceloop-sdk==0.40.4 ; python_version >= "3.10" and python_version < "4.0"