Skip to content

Commit 6e248e4

Browse files
mehulsonowalsabrenner
mehulsonowal
and
sabrenner
committed
feat(llmobs): emit experiment traces (#9479)
* feat(llmobs): add experiments dataset operations * feat(llmobs): add CSV dataset creation * chore(llmobs): use csv parser for dataset import * chore(llmobs): add csv parser license * feat(llmobs): include dataset version in experiments * fix(llmobs): address dataset review feedback * create jsontype to replace any * remove dataset operations from llmobs * remove csv dataset func * noops produce warn logs instead of error throwing * use vcr testing instead * Update packages/dd-trace/src/llmobs/experiments/index.js Co-authored-by: Sam Brenner <106700075+sabrenner@users.noreply.github.com> * fix(llmobs): address experiments ci failures * test(llmobs): cover experiments noop accessors * fix(llmobs): refresh experiments facade on config changes * feat(llmobs): emit experiment traces * fix(llmobs): preserve dataset record ids for experiments * create utils, double check comments * fix(llmobs): align experiments noop tests * fix(llmobs): address experiment review feedback * fix(llmobs): remove async experiment alias * address comments * fix(llmobs): align experiment type definitions * Update packages/dd-trace/src/llmobs/experiments/experiment.js Co-authored-by: Sam Brenner <106700075+sabrenner@users.noreply.github.com> * assign output to task execution * revert span condition * revert tagging changes --------- Co-authored-by: Sam Brenner <106700075+sabrenner@users.noreply.github.com>
1 parent a5ea714 commit 6e248e4

15 files changed

Lines changed: 965 additions & 138 deletions

File tree

‎index.d.ts‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3742,19 +3742,31 @@ declare namespace tracer {
37423742
*/
37433743
type ExperimentTask = (
37443744
input: JSONType,
3745-
config: Record<string, JSONType>
3745+
config: Record<string, JSONType>,
3746+
metadata?: Record<string, JSONType>
37463747
) => JSONType | Promise<JSONType>
37473748

37483749
/**
37493750
* Scores a single task output. The return type selects the metric:
3750-
* `boolean` -> boolean, `number` -> score, anything else -> categorical.
3751+
* `boolean` -> boolean, `number` -> score, `string` -> categorical, anything else -> json.
37513752
*/
37523753
type ExperimentEvaluator = (
37533754
input: JSONType,
37543755
output: JSONType,
37553756
expectedOutput: JSONType
37563757
) => JSONType | Promise<JSONType>
37573758

3759+
/**
3760+
* Scores all rows in an experiment run and emits a summary metric.
3761+
*/
3762+
type ExperimentSummaryEvaluator = (
3763+
inputs: any[],
3764+
outputs: any[],
3765+
expectedOutputs: any[],
3766+
evaluatorResults: Record<string, any[]>,
3767+
metadata?: Array<Record<string, any>>
3768+
) => any | Promise<any>
3769+
37583770
interface CreateDatasetOptions {
37593771
description?: string
37603772
records?: Array<{
@@ -3769,13 +3781,24 @@ declare namespace tracer {
37693781
name: string
37703782
dataset: Dataset
37713783
task: ExperimentTask
3772-
/** Evaluators keyed by metric label. */
3773-
evaluators?: Record<string, ExperimentEvaluator>
3784+
/** Evaluators keyed by metric label, or named functions. */
3785+
evaluators?: Record<string, ExperimentEvaluator> | ExperimentEvaluator[]
3786+
/** Summary evaluators keyed by metric label, or named functions. */
3787+
summaryEvaluators?: Record<string, ExperimentSummaryEvaluator> | ExperimentSummaryEvaluator[]
37743788
description?: string
37753789
config?: Record<string, JSONType>
37763790
tags?: Record<string, string>
37773791
}
37783792

3793+
interface ExperimentRunOptions {
3794+
/** Maximum retries for task and evaluator failures. Default 0. */
3795+
maxRetries?: number
3796+
/** Delay before a retry, in milliseconds. Default 100 * (attempt + 1). */
3797+
retryDelay?: (attempt: number) => number
3798+
/** Reject on the first task/evaluator error instead of capturing it. Default false. */
3799+
throwOnErrors?: boolean
3800+
}
3801+
37793802
interface PullDatasetOptions {
37803803
/** Dataset version to pull. Defaults to latest. */
37813804
version?: number
@@ -3801,9 +3824,20 @@ declare namespace tracer {
38013824
evaluationErrors: Record<string, string>
38023825
}
38033826

3827+
interface ExperimentRun {
3828+
runId: string
3829+
runIteration: number
3830+
rows: ExperimentResultRow[]
3831+
summaryEvaluations: Record<string, { value: any, error: string | null }>
3832+
}
3833+
38043834
interface ExperimentResult {
38053835
experimentId: string
38063836
rows: ExperimentResultRow[]
3837+
/** Single-run summary evaluator results. */
3838+
summaryEvaluations: Record<string, { value: any, error: string | null }>
3839+
/** Experiment runs. P0 Node experiments currently return one run. */
3840+
runs: ExperimentRun[]
38073841
/** Dashboard URL for the experiment. */
38083842
url: string
38093843
}
@@ -3838,7 +3872,7 @@ declare namespace tracer {
38383872
name (): string
38393873
experimentId (): string | null
38403874
url (): string | null
3841-
run (): Promise<ExperimentResult>
3875+
run (options?: ExperimentRunOptions): Promise<ExperimentResult>
38423876
}
38433877

38443878
interface Experiments {

‎index.d.v5.ts‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3939,19 +3939,31 @@ declare namespace tracer {
39393939
*/
39403940
type ExperimentTask = (
39413941
input: JSONType,
3942-
config: Record<string, JSONType>
3942+
config: Record<string, JSONType>,
3943+
metadata?: Record<string, JSONType>
39433944
) => JSONType | Promise<JSONType>
39443945

39453946
/**
39463947
* Scores a single task output. The return type selects the metric:
3947-
* `boolean` -> boolean, `number` -> score, anything else -> categorical.
3948+
* `boolean` -> boolean, `number` -> score, `string` -> categorical, anything else -> json.
39483949
*/
39493950
type ExperimentEvaluator = (
39503951
input: JSONType,
39513952
output: JSONType,
39523953
expectedOutput: JSONType
39533954
) => JSONType | Promise<JSONType>
39543955

3956+
/**
3957+
* Scores all rows in an experiment run and emits a summary metric.
3958+
*/
3959+
type ExperimentSummaryEvaluator = (
3960+
inputs: any[],
3961+
outputs: any[],
3962+
expectedOutputs: any[],
3963+
evaluatorResults: Record<string, any[]>,
3964+
metadata?: Array<Record<string, any>>
3965+
) => any | Promise<any>
3966+
39553967
interface CreateDatasetOptions {
39563968
description?: string
39573969
records?: Array<{
@@ -3966,13 +3978,24 @@ declare namespace tracer {
39663978
name: string
39673979
dataset: Dataset
39683980
task: ExperimentTask
3969-
/** Evaluators keyed by metric label. */
3970-
evaluators?: Record<string, ExperimentEvaluator>
3981+
/** Evaluators keyed by metric label, or named functions. */
3982+
evaluators?: Record<string, ExperimentEvaluator> | ExperimentEvaluator[]
3983+
/** Summary evaluators keyed by metric label, or named functions. */
3984+
summaryEvaluators?: Record<string, ExperimentSummaryEvaluator> | ExperimentSummaryEvaluator[]
39713985
description?: string
39723986
config?: Record<string, JSONType>
39733987
tags?: Record<string, string>
39743988
}
39753989

3990+
interface ExperimentRunOptions {
3991+
/** Maximum retries for task and evaluator failures. Default 0. */
3992+
maxRetries?: number
3993+
/** Delay before a retry, in milliseconds. Default 100 * (attempt + 1). */
3994+
retryDelay?: (attempt: number) => number
3995+
/** Reject on the first task/evaluator error instead of capturing it. Default false. */
3996+
throwOnErrors?: boolean
3997+
}
3998+
39763999
interface PullDatasetOptions {
39774000
/** Dataset version to pull. Defaults to latest. */
39784001
version?: number
@@ -3998,9 +4021,20 @@ declare namespace tracer {
39984021
evaluationErrors: Record<string, string>
39994022
}
40004023

4024+
interface ExperimentRun {
4025+
runId: string
4026+
runIteration: number
4027+
rows: ExperimentResultRow[]
4028+
summaryEvaluations: Record<string, { value: any, error: string | null }>
4029+
}
4030+
40014031
interface ExperimentResult {
40024032
experimentId: string
40034033
rows: ExperimentResultRow[]
4034+
/** Single-run summary evaluator results. */
4035+
summaryEvaluations: Record<string, { value: any, error: string | null }>
4036+
/** Experiment runs. P0 Node experiments currently return one run. */
4037+
runs: ExperimentRun[]
40044038
/** Dashboard URL for the experiment. */
40054039
url: string
40064040
}
@@ -4035,7 +4069,7 @@ declare namespace tracer {
40354069
name (): string
40364070
experimentId (): string | null
40374071
url (): string | null
4038-
run (): Promise<ExperimentResult>
4072+
run (options?: ExperimentRunOptions): Promise<ExperimentResult>
40394073
}
40404074

40414075
interface Experiments {

‎packages/dd-trace/src/llmobs/constants/tags.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
'use strict'
22

33
module.exports = {
4-
SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'],
4+
SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval', 'experiment'],
55
SPAN_KIND: '_ml_obs.meta.span.kind',
66
SESSION_ID: '_ml_obs.session_id',
77
SESSION_ID_TRACE_DEFAULT_KEY: '_ml_obs.trace_session_id',

0 commit comments

Comments
 (0)