feat(scores): add session and dataset run scores (#1201)

Author: hassiebp

langfuse/_client/client.py

@@ -166,7 +166,9 @@ def __init__(
         self._project_id = None
         sample_rate = sample_rate or float(os.environ.get(LANGFUSE_SAMPLE_RATE, 1.0))
         if not 0.0 <= sample_rate <= 1.0:
-            raise ValueError(f"Sample rate must be between 0.0 and 1.0, got {sample_rate}")
+            raise ValueError(
+                f"Sample rate must be between 0.0 and 1.0, got {sample_rate}"
+            )
 
         self._tracing_enabled = (
             tracing_enabled
@@ -1214,12 +1216,15 @@ def create_score(
         *,
         name: str,
         value: float,
-        trace_id: str,
+        session_id: Optional[str] = None,
+        dataset_run_id: Optional[str] = None,
+        trace_id: Optional[str] = None,
         observation_id: Optional[str] = None,
         score_id: Optional[str] = None,
         data_type: Optional[Literal["NUMERIC", "BOOLEAN"]] = None,
         comment: Optional[str] = None,
         config_id: Optional[str] = None,
+        metadata: Optional[Any] = None,
     ) -> None: ...
 
     @overload
@@ -1228,25 +1233,31 @@ def create_score(
         *,
         name: str,
         value: str,
-        trace_id: str,
+        session_id: Optional[str] = None,
+        dataset_run_id: Optional[str] = None,
+        trace_id: Optional[str] = None,
         score_id: Optional[str] = None,
         observation_id: Optional[str] = None,
         data_type: Optional[Literal["CATEGORICAL"]] = "CATEGORICAL",
         comment: Optional[str] = None,
         config_id: Optional[str] = None,
+        metadata: Optional[Any] = None,
     ) -> None: ...
 
     def create_score(
         self,
         *,
         name: str,
         value: Union[float, str],
-        trace_id: str,
+        session_id: Optional[str] = None,
+        dataset_run_id: Optional[str] = None,
+        trace_id: Optional[str] = None,
         observation_id: Optional[str] = None,
         score_id: Optional[str] = None,
         data_type: Optional[ScoreDataType] = None,
         comment: Optional[str] = None,
         config_id: Optional[str] = None,
+        metadata: Optional[Any] = None,
     ) -> None:
         """Create a score for a specific trace or observation.
 
@@ -1256,12 +1267,15 @@ def create_score(
         Args:
             name: Name of the score (e.g., "relevance", "accuracy")
             value: Score value (can be numeric for NUMERIC/BOOLEAN types or string for CATEGORICAL)
+            session_id: ID of the Langfuse session to associate the score with
+            dataset_run_id: ID of the Langfuse dataset run to associate the score with
             trace_id: ID of the Langfuse trace to associate the score with
-            observation_id: Optional ID of the specific observation to score
+            observation_id: Optional ID of the specific observation to score. Trace ID must be provided too.
             score_id: Optional custom ID for the score (auto-generated if not provided)
             data_type: Type of score (NUMERIC, BOOLEAN, or CATEGORICAL)
             comment: Optional comment or explanation for the score
             config_id: Optional ID of a score config defined in Langfuse
+            metadata: Optional metadata to be attached to the score
 
         Example:
             ```python
@@ -1292,6 +1306,8 @@ def create_score(
         try:
             score_event = {
                 "id": score_id,
+                "session_id": session_id,
+                "dataset_run_id": dataset_run_id,
                 "trace_id": trace_id,
                 "observation_id": observation_id,
                 "name": name,
@@ -1300,6 +1316,7 @@ def create_score(
                 "comment": comment,
                 "config_id": config_id,
                 "environment": self._environment,
+                "metadata": metadata,
             }
 
             new_body = ScoreBody(**score_event)

langfuse/_client/resource_manager.py

@@ -256,13 +256,18 @@ def add_score_task(self, event: dict):
             # Sample scores with the same sampler that is used for tracing
             tracer_provider = cast(TracerProvider, otel_trace_api.get_tracer_provider())
             should_sample = (
-                tracer_provider.sampler.should_sample(
-                    parent_context=None,
-                    trace_id=int(event["body"].trace_id, 16),
-                    name="score",
-                ).decision
-                == Decision.RECORD_AND_SAMPLE
-                if hasattr(event["body"], "trace_id")
+                (
+                    tracer_provider.sampler.should_sample(
+                        parent_context=None,
+                        trace_id=int(event["body"].trace_id, 16),
+                        name="score",
+                    ).decision
+                    == Decision.RECORD_AND_SAMPLE
+                    if hasattr(event["body"], "trace_id")
+                    else True
+                )
+                if event["body"].trace_id
+                is not None  # do not sample out session / dataset run scores
                 else True
             )
 

langfuse/api/__init__.py

@@ -6,11 +6,20 @@
     AnnotationQueueItem,
     AnnotationQueueObjectType,
     AnnotationQueueStatus,
+    ApiKeyDeletionResponse,
+    ApiKeyList,
+    ApiKeyResponse,
+    ApiKeySummary,
+    AuthenticationScheme,
     BaseEvent,
     BasePrompt,
     BaseScore,
+    BaseScoreV1,
     BooleanScore,
+    BooleanScoreV1,
+    BulkConfig,
     CategoricalScore,
+    CategoricalScoreV1,
     ChatMessage,
     ChatPrompt,
     Comment,
@@ -39,8 +48,6 @@
     CreateSpanBody,
     CreateSpanEvent,
     CreateTextPromptRequest,
-    DailyMetrics,
-    DailyMetricsDetails,
     Dataset,
     DatasetItem,
     DatasetRun,
@@ -51,7 +58,9 @@
     DeleteDatasetItemResponse,
     DeleteDatasetRunResponse,
     DeleteTraceResponse,
+    EmptyResponse,
     Error,
+    FilterConfig,
     GetCommentsResponse,
     GetMediaResponse,
     GetMediaUploadUrlRequest,
@@ -83,11 +92,18 @@
     IngestionUsage,
     MapValue,
     MediaContentType,
+    MembershipRequest,
+    MembershipResponse,
+    MembershipRole,
+    MembershipsResponse,
     MethodNotAllowedError,
+    MetricsResponse,
     Model,
+    ModelPrice,
     ModelUsageUnit,
     NotFoundError,
     NumericScore,
+    NumericScoreV1,
     Observation,
     ObservationBody,
     ObservationLevel,
@@ -99,33 +115,53 @@
     OpenAiResponseUsageSchema,
     OpenAiUsage,
     OptionalObservationBody,
+    OrganizationProject,
+    OrganizationProjectsResponse,
     PaginatedAnnotationQueueItems,
     PaginatedAnnotationQueues,
     PaginatedDatasetItems,
+    PaginatedDatasetRunItems,
     PaginatedDatasetRuns,
     PaginatedDatasets,
     PaginatedModels,
     PaginatedSessions,
     PatchMediaBody,
     Project,
+    ProjectDeletionResponse,
     Projects,
     Prompt,
     PromptMeta,
     PromptMetaListResponse,
     Prompt_Chat,
     Prompt_Text,
+    ResourceMeta,
+    ResourceType,
+    ResourceTypesResponse,
+    SchemaExtension,
+    SchemaResource,
+    SchemasResponse,
+    ScimEmail,
+    ScimFeatureSupport,
+    ScimName,
+    ScimUser,
+    ScimUsersListResponse,
     Score,
     ScoreBody,
     ScoreConfig,
     ScoreConfigs,
     ScoreDataType,
     ScoreEvent,
     ScoreSource,
+    ScoreV1,
+    ScoreV1_Boolean,
+    ScoreV1_Categorical,
+    ScoreV1_Numeric,
     Score_Boolean,
     Score_Categorical,
     Score_Numeric,
     SdkLogBody,
     SdkLogEvent,
+    ServiceProviderConfig,
     ServiceUnavailableError,
     Session,
     SessionWithTraces,
@@ -146,8 +182,8 @@
     UpdateSpanBody,
     UpdateSpanEvent,
     Usage,
-    UsageByModel,
     UsageDetails,
+    UserMeta,
     annotation_queues,
     comments,
     commons,
@@ -160,11 +196,14 @@
     metrics,
     models,
     observations,
+    organizations,
     projects,
     prompt_version,
     prompts,
+    scim,
     score,
     score_configs,
+    score_v_2,
     sessions,
     trace,
     utils,
@@ -176,11 +215,20 @@
     "AnnotationQueueItem",
     "AnnotationQueueObjectType",
     "AnnotationQueueStatus",
+    "ApiKeyDeletionResponse",
+    "ApiKeyList",
+    "ApiKeyResponse",
+    "ApiKeySummary",
+    "AuthenticationScheme",
     "BaseEvent",
     "BasePrompt",
     "BaseScore",
+    "BaseScoreV1",
     "BooleanScore",
+    "BooleanScoreV1",
+    "BulkConfig",
     "CategoricalScore",
+    "CategoricalScoreV1",
     "ChatMessage",
     "ChatPrompt",
     "Comment",
@@ -209,8 +257,6 @@
     "CreateSpanBody",
     "CreateSpanEvent",
     "CreateTextPromptRequest",
-    "DailyMetrics",
-    "DailyMetricsDetails",
     "Dataset",
     "DatasetItem",
     "DatasetRun",
@@ -221,7 +267,9 @@
     "DeleteDatasetItemResponse",
     "DeleteDatasetRunResponse",
     "DeleteTraceResponse",
+    "EmptyResponse",
     "Error",
+    "FilterConfig",
     "GetCommentsResponse",
     "GetMediaResponse",
     "GetMediaUploadUrlRequest",
@@ -253,11 +301,18 @@
     "IngestionUsage",
     "MapValue",
     "MediaContentType",
+    "MembershipRequest",
+    "MembershipResponse",
+    "MembershipRole",
+    "MembershipsResponse",
     "MethodNotAllowedError",
+    "MetricsResponse",
     "Model",
+    "ModelPrice",
     "ModelUsageUnit",
     "NotFoundError",
     "NumericScore",
+    "NumericScoreV1",
     "Observation",
     "ObservationBody",
     "ObservationLevel",
@@ -269,33 +324,53 @@
     "OpenAiResponseUsageSchema",
     "OpenAiUsage",
     "OptionalObservationBody",
+    "OrganizationProject",
+    "OrganizationProjectsResponse",
     "PaginatedAnnotationQueueItems",
     "PaginatedAnnotationQueues",
     "PaginatedDatasetItems",
+    "PaginatedDatasetRunItems",
     "PaginatedDatasetRuns",
     "PaginatedDatasets",
     "PaginatedModels",
     "PaginatedSessions",
     "PatchMediaBody",
     "Project",
+    "ProjectDeletionResponse",
     "Projects",
     "Prompt",
     "PromptMeta",
     "PromptMetaListResponse",
     "Prompt_Chat",
     "Prompt_Text",
+    "ResourceMeta",
+    "ResourceType",
+    "ResourceTypesResponse",
+    "SchemaExtension",
+    "SchemaResource",
+    "SchemasResponse",
+    "ScimEmail",
+    "ScimFeatureSupport",
+    "ScimName",
+    "ScimUser",
+    "ScimUsersListResponse",
     "Score",
     "ScoreBody",
     "ScoreConfig",
     "ScoreConfigs",
     "ScoreDataType",
     "ScoreEvent",
     "ScoreSource",
+    "ScoreV1",
+    "ScoreV1_Boolean",
+    "ScoreV1_Categorical",
+    "ScoreV1_Numeric",
     "Score_Boolean",
     "Score_Categorical",
     "Score_Numeric",
     "SdkLogBody",
     "SdkLogEvent",
+    "ServiceProviderConfig",
     "ServiceUnavailableError",
     "Session",
     "SessionWithTraces",
@@ -316,8 +391,8 @@
     "UpdateSpanBody",
     "UpdateSpanEvent",
     "Usage",
-    "UsageByModel",
     "UsageDetails",
+    "UserMeta",
     "annotation_queues",
     "comments",
     "commons",
@@ -330,11 +405,14 @@
     "metrics",
     "models",
     "observations",
+    "organizations",
     "projects",
     "prompt_version",
     "prompts",
+    "scim",
     "score",
     "score_configs",
+    "score_v_2",
     "sessions",
     "trace",
     "utils",