diff --git a/eval/classifiers.go b/eval/classifiers.go new file mode 100644 index 00000000..1530d866 --- /dev/null +++ b/eval/classifiers.go @@ -0,0 +1,60 @@ +package eval + +import ( + "context" +) + +// Classifier is an interface for categorizing the output of a task. +// +// Unlike [Scorer], which returns numeric scores, a Classifier returns one or +// more [Classification] items with structured id/label/metadata. +type Classifier[I, R any] interface { + // Name returns the name of this classifier. + Name() string + // Run categorizes the task result. + // It returns zero or more Classification results. + Run(context.Context, TaskResult[I, R]) (Classifications, error) +} + +// Classification represents a single category result returned by a Classifier. +type Classification struct { + // Name is the grouping key for this classification. + // If empty, it defaults to the name of the Classifier that produced it. + Name string + + // ID is the stable identifier for the category (required). + ID string + + // Label is an optional human-readable label. + Label string + + // Metadata is optional additional metadata for this classification. + Metadata map[string]interface{} +} + +// Classifications is a collection of Classification results returned by classifiers. +type Classifications = []Classification + +// ClassifyFunc is a function that classifies a task result and returns a list of Classifications. +type ClassifyFunc[I, R any] func(ctx context.Context, result TaskResult[I, R]) (Classifications, error) + +// NewClassifier creates a new classifier with the given name and classify function. +func NewClassifier[I, R any](name string, fn ClassifyFunc[I, R]) Classifier[I, R] { + return &classifierImpl[I, R]{ + name: name, + fn: fn, + } +} + +type classifierImpl[I, R any] struct { + name string + fn ClassifyFunc[I, R] +} + +func (c *classifierImpl[I, R]) Name() string { + return c.name +} + +func (c *classifierImpl[I, R]) Run(ctx context.Context, result TaskResult[I, R]) (Classifications, error) { + return c.fn(ctx, result) +} diff --git a/eval/classifiers_test.go b/eval/classifiers_test.go new file mode 100644 index 00000000..a41499d8 --- /dev/null +++ b/eval/classifiers_test.go @@ -0,0 +1,374 @@ +package eval + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/codes" + + "github.com/braintrustdata/braintrust-sdk-go/internal/oteltest" +) + +// simpleClassifier is a test Classifier that returns a fixed set of +// classifications, optionally with a forced error. +type simpleClassifier struct { + name string + classifications Classifications + err error +} + +func (c *simpleClassifier) Name() string { return c.name } + +func (c *simpleClassifier) Run(_ context.Context, _ TaskResult[testInput, testOutput]) (Classifications, error) { + if c.err != nil { + return nil, c.err + } + return c.classifications, nil +} + +// findSpanByName returns the first span matching name. Spans from parallel +// scorer/classifier passes may be interleaved, so name-based lookup is the +// stable way to assert on them. +func findSpanByName(t *testing.T, spans []oteltest.Span, name string) *oteltest.Span { + t.Helper() + for i := range spans { + if spans[i].Name() == name { + return &spans[i] + } + } + t.Fatalf("no span with name %q (got: %s)", name, spanNames(spans)) + return nil +} + +func spanNames(spans []oteltest.Span) string { + names := make([]string, len(spans)) + for i := range spans { + names[i] = spans[i].Name() + } + return "[" + joinStrings(names, ", ") + "]" +} + +func joinStrings(parts []string, sep string) string { + out := "" + for i, p := range parts { + if i > 0 { + out += sep + } + out += p + } + return out +} + +func TestNewEval_Classifier_SingleLabel(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}, Expected: testOutput{Result: "greeting"}}, + }) + task := T(func(_ context.Context, input testInput) (testOutput, error) { + return testOutput{Result: "greeting-" + input.Value}, nil + }) + + classifier := &simpleClassifier{ + name: "category", + classifications: Classifications{ + {ID: "greeting", Label: "Greeting"}, + }, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, nil, []Classifier[testInput, testOutput]{classifier}, 1) + + _, err := ute.eval.run(context.Background()) + require.NoError(t, err) + + spans := ute.exporter.Flush() + // Per case: task + classifier + eval = 3 spans. 1 case = 3 spans. + require.Len(t, spans, 3) + + classifierSpan := findSpanByName(t, spans, "category") + classifierSpan.AssertJSONAttrEquals("braintrust.span_attributes", map[string]any{ + "type": "classifier", + "name": "category", + "purpose": "scorer", + }) + classifierSpan.AssertJSONAttrEquals("braintrust.input_json", map[string]any{ + "input": map[string]any{"value": "hi"}, + "expected": map[string]any{"result": "greeting"}, + "output": map[string]any{"result": "greeting-hi"}, + }) + classifierSpan.AssertJSONAttrEquals("braintrust.output_json", map[string]any{ + "category": []any{ + map[string]any{"id": "greeting", "label": "Greeting"}, + }, + }) + + evalSpan := findSpanByName(t, spans, "eval") + evalSpan.AssertJSONAttrEquals("braintrust.classifications", map[string]any{ + "category": []any{ + map[string]any{"id": "greeting", "label": "Greeting"}, + }, + }) +} + +func TestNewEval_Classifier_MultiLabel(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "please send it immediately"}}, + }) + task := T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }) + + // A classifier returning multiple classifications with the same Name + // should aggregate as a slice keyed by that name. + classifier := &simpleClassifier{ + name: "tone", + classifications: Classifications{ + {Name: "tone", ID: "urgent", Label: "Urgent"}, + {Name: "tone", ID: "polite", Label: "Polite"}, + }, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, nil, []Classifier[testInput, testOutput]{classifier}, 1) + + _, err := ute.eval.run(context.Background()) + require.NoError(t, err) + + spans := ute.exporter.Flush() + require.Len(t, spans, 3) + + expected := map[string]any{ + "tone": []any{ + map[string]any{"id": "urgent", "label": "Urgent"}, + map[string]any{"id": "polite", "label": "Polite"}, + }, + } + findSpanByName(t, spans, "tone").AssertJSONAttrEquals("braintrust.output_json", expected) + findSpanByName(t, spans, "eval").AssertJSONAttrEquals("braintrust.classifications", expected) +} + +func TestNewEval_Classifier_NameDefaulting(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}}, + }) + task := T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }) + + // Classification.Name left empty — should default to the classifier name. + classifier := &simpleClassifier{ + name: "category", + classifications: Classifications{ + {ID: "greeting"}, // no Name + }, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, nil, []Classifier[testInput, testOutput]{classifier}, 1) + + _, err := ute.eval.run(context.Background()) + require.NoError(t, err) + + spans := ute.exporter.Flush() + findSpanByName(t, spans, "eval").AssertJSONAttrEquals("braintrust.classifications", map[string]any{ + "category": []any{ + map[string]any{"id": "greeting"}, + }, + }) +} + +func TestNewEval_Classifier_MultipleClassifiers(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}}, + }) + task := T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }) + + classifiers := []Classifier[testInput, testOutput]{ + &simpleClassifier{ + name: "category", + classifications: Classifications{ + {ID: "greeting", Label: "Greeting"}, + }, + }, + &simpleClassifier{ + name: "sentiment", + classifications: Classifications{ + {ID: "positive", Metadata: map[string]any{"confidence": 0.9}}, + }, + }, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, nil, classifiers, 1) + + _, err := ute.eval.run(context.Background()) + require.NoError(t, err) + + spans := ute.exporter.Flush() + findSpanByName(t, spans, "eval").AssertJSONAttrEquals("braintrust.classifications", map[string]any{ + "category": []any{ + map[string]any{"id": "greeting", "label": "Greeting"}, + }, + "sentiment": []any{ + map[string]any{"id": "positive", "metadata": map[string]any{"confidence": 0.9}}, + }, + }) +} + +func TestNewEval_Classifier_ErrorIsNonFatal(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}, Metadata: map[string]any{"caseKey": "caseVal"}}, + }) + task := T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }) + + boom := errors.New("classifier exploded") + classifiers := []Classifier[testInput, testOutput]{ + &simpleClassifier{ + name: "ok-classifier", + classifications: Classifications{ + {ID: "yes", Label: "Yes"}, + }, + }, + &simpleClassifier{name: "broken", err: boom}, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, nil, classifiers, 1) + + result, err := ute.eval.run(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "classifier exploded") + assert.NotNil(t, result) + + spans := ute.exporter.Flush() + + // The successful classifier still ran and contributed to the aggregated map. + findSpanByName(t, spans, "ok-classifier").AssertJSONAttrEquals("braintrust.output_json", map[string]any{ + "ok-classifier": []any{ + map[string]any{"id": "yes", "label": "Yes"}, + }, + }) + + // The broken classifier's span has Error status with an exception event. + brokenSpan := findSpanByName(t, spans, "broken") + assert.Equal(t, codes.Error, brokenSpan.Status().Code) + events := brokenSpan.Events() + require.Len(t, events, 1) + assert.Equal(t, "exception", events[0].Name) + + // The eval span carries the aggregated classifications and the + // classifier_errors merged into the case metadata. + evalSpan := findSpanByName(t, spans, "eval") + evalSpan.AssertJSONAttrEquals("braintrust.classifications", map[string]any{ + "ok-classifier": []any{ + map[string]any{"id": "yes", "label": "Yes"}, + }, + }) + evalSpan.AssertJSONAttrEquals("braintrust.metadata", map[string]any{ + "caseKey": "caseVal", + "classifier_errors": map[string]any{ + "broken": "classifier error: classifier \"broken\" failed: classifier exploded", + }, + }) +} + +func TestNewEval_Classifier_EmptyIDIsError(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}}, + }) + task := T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }) + + classifier := &simpleClassifier{ + name: "category", + classifications: Classifications{ + {ID: ""}, // empty ID is not allowed per spec + }, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, nil, []Classifier[testInput, testOutput]{classifier}, 1) + + _, err := ute.eval.run(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty ID") +} + +func TestNewEval_ScorerAndClassifier_RunBoth(t *testing.T) { + t.Parallel() + + cases := NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}, Expected: testOutput{Result: "ok"}}, + }) + task := T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }) + + scorers := []Scorer[testInput, testOutput]{ + &simpleScorer{name: "accuracy", score: 1.0}, + } + classifiers := []Classifier[testInput, testOutput]{ + &simpleClassifier{ + name: "category", + classifications: Classifications{ + {ID: "greeting", Label: "Greeting"}, + }, + }, + } + + ute := newUnitTestEvalWithClassifiers(t, cases, task, scorers, classifiers, 1) + + _, err := ute.eval.run(context.Background()) + require.NoError(t, err) + + spans := ute.exporter.Flush() + // task + accuracy + category + eval = 4 spans. Their order between + // accuracy/category may interleave because scorers and classifiers run + // in parallel goroutines, so look them up by name. + require.Len(t, spans, 4) + + findSpanByName(t, spans, "accuracy").AssertJSONAttrEquals("braintrust.scores", map[string]any{ + "accuracy": 1.0, + }) + findSpanByName(t, spans, "category").AssertJSONAttrEquals("braintrust.output_json", map[string]any{ + "category": []any{ + map[string]any{"id": "greeting", "label": "Greeting"}, + }, + }) +} + +func TestRun_RequiresScorerOrClassifier(t *testing.T) { + t.Parallel() + + // Going through the public run() path: at least one of Scorers or + // Classifiers must be non-empty. + _, err := run[testInput, testOutput]( + context.Background(), + Opts[testInput, testOutput]{ + Experiment: "exp", + Dataset: NewDataset([]Case[testInput, testOutput]{ + {Input: testInput{Value: "hi"}}, + }), + Task: T(func(_ context.Context, _ testInput) (testOutput, error) { + return testOutput{Result: "ok"}, nil + }), + // no Scorers, no Classifiers + }, + nil, nil, nil, "some-project", + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "at least one of Scorers or Classifiers") +} diff --git a/eval/eval.go b/eval/eval.go index 3f3b04f5..bd5fae63 100644 --- a/eval/eval.go +++ b/eval/eval.go @@ -49,6 +49,7 @@ var ( // Private error variables (users don't need to check these) errEval = errors.New("eval error") errScorer = errors.New("scorer error") + errClassifier = errors.New("classifier error") errTaskRun = errors.New("task run error") errCaseIterator = errors.New("case iterator error") ) @@ -75,7 +76,12 @@ type Opts[I, R any] struct { Experiment string Dataset Dataset[I, R] Task TaskFunc[I, R] - Scorers []Scorer[I, R] + + // At least one of Scorers or Classifiers must be provided. + // Scorers return numeric scores; Classifiers return categorical labels. + // They run in parallel for each case and are independent of each other. + Scorers []Scorer[I, R] + Classifiers []Classifier[I, R] // Optional ProjectName string // Project name (uses default from config if not specified) @@ -230,6 +236,7 @@ type eval[I, R any] struct { datasetID string // For origin.object_id task TaskFunc[I, R] scorers []Scorer[I, R] + classifiers []Classifier[I, R] tracer oteltrace.Tracer startSpanOpt oteltrace.SpanStartOption goroutines int @@ -254,6 +261,7 @@ func newEval[I, R any]( dataset Dataset[I, R], task TaskFunc[I, R], scorers []Scorer[I, R], + classifiers []Classifier[I, R], parallelism int, quiet bool, ) *eval[I, R] { @@ -281,6 +289,7 @@ func newEval[I, R any]( datasetID: datasetID, task: task, scorers: scorers, + classifiers: classifiers, tracer: tracer, startSpanOpt: startSpanOpt, goroutines: goroutines, @@ -320,6 +329,7 @@ func newEvalOpts[I, R any](ctx context.Context, s *auth.Session, tp *trace.Trace opts.Dataset, opts.Task, opts.Scorers, + opts.Classifiers, opts.Parallelism, opts.Quiet, ), nil @@ -449,12 +459,34 @@ func (e *eval[I, R]) runCase(ctx context.Context, span oteltrace.Span, c Case[I, return err } - _, err = e.runScorers(ctx, taskResult) - if err != nil { - span.SetStatus(codes.Error, err.Error()) - return err + // Run scorers and classifiers in parallel. Each pass is independent — + // a failing scorer must not block classifiers (and vice versa), and + // per-pass errors are aggregated rather than fatal. + var ( + scorerErr error + classifierErr error + wg sync.WaitGroup + ) + if len(e.scorers) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + _, scorerErr = e.runScorers(ctx, taskResult) + }() + } + if len(e.classifiers) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + classifierErr = e.runClassifiers(ctx, span, c, taskResult) + }() } + wg.Wait() + if joined := errors.Join(scorerErr, classifierErr); joined != nil { + span.SetStatus(codes.Error, joined.Error()) + return joined + } return nil } @@ -611,6 +643,159 @@ func (e *eval[I, R]) runScorer(ctx context.Context, scorer Scorer[I, R], taskRes return scores, nil } +// runClassifiers runs each classifier concurrently in its own span, +// aggregates the resulting classifications by name onto the eval span, +// and records any per-classifier errors as non-fatal aggregate errors. +// +// Per the cross-SDK spec, classifier failures must not abort the eval +// or affect other classifiers/scorers; they are surfaced via +// braintrust.metadata.classifier_errors on the eval span and joined into +// the returned error. +func (e *eval[I, R]) runClassifiers(ctx context.Context, evalSpan oteltrace.Span, c Case[I, R], taskResult TaskResult[I, R]) error { + // Run all classifiers in parallel. Each goroutine writes to its own + // slot in `results`, so no synchronization is needed during execution. + // We merge results sequentially below in declaration order to keep + // cross-classifier aggregation deterministic. + type classifierResult struct { + spanName string + items []classificationItem + err error + } + results := make([]classifierResult, len(e.classifiers)) + var wg sync.WaitGroup + for i, classifier := range e.classifiers { + wg.Add(1) + go func(i int, classifier Classifier[I, R]) { + defer wg.Done() + spanName := classifier.Name() + if spanName == "" { + spanName = fmt.Sprintf("classifier_%d", i) + } + items, err := e.runClassifier(ctx, spanName, classifier, taskResult) + results[i] = classifierResult{spanName: spanName, items: items, err: err} + }(i, classifier) + } + wg.Wait() + + aggregated := make(map[string][]map[string]any) + classifierErrors := make(map[string]string) + var errs []error + + for _, r := range results { + if r.err != nil { + classifierErrors[r.spanName] = r.err.Error() + errs = append(errs, r.err) + continue + } + + // Group items by their resolved name. Items keep their stable order; + // duplicates with the same {name, id} are preserved. + for _, item := range r.items { + aggregated[item.name] = append(aggregated[item.name], item.payload) + } + } + + // Log aggregated classifications onto the eval span, if any. + if len(aggregated) > 0 { + if err := setJSONAttr(evalSpan, "braintrust.classifications", aggregated); err != nil { + errs = append(errs, err) + } + } + + // Merge classifier_errors into the eval span's metadata. The eval span's + // metadata was set in runCase before classifiers ran, so we rewrite it + // with the merged map. + if len(classifierErrors) > 0 { + merged := make(map[string]any, len(c.Metadata)+1) + for k, v := range c.Metadata { + merged[k] = v + } + merged["classifier_errors"] = classifierErrors + if err := setJSONAttr(evalSpan, "braintrust.metadata", merged); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + +// classificationItem is the wire-format payload (id/label?/metadata?) paired +// with the resolved grouping name, used internally during aggregation. +type classificationItem struct { + name string + payload map[string]any +} + +// runClassifier executes a single classifier in its own span. It mirrors +// runScorer: a child span named after the classifier, with input_json set +// to the task result, and the classifier's output written to output_json. +func (e *eval[I, R]) runClassifier(ctx context.Context, spanName string, classifier Classifier[I, R], taskResult TaskResult[I, R]) ([]classificationItem, error) { + ctx, span := e.tracer.Start(ctx, spanName, e.startSpanOpt) + defer span.End() + + spanAttrs := map[string]any{ + "type": "classifier", + "name": spanName, + "purpose": "scorer", + } + if err := setJSONAttr(span, "braintrust.span_attributes", spanAttrs); err != nil { + return nil, err + } + + // Log the task result (input/expected/output) as the classifier's input. + classifierInput := map[string]any{ + "input": taskResult.Input, + "expected": taskResult.Expected, + "output": taskResult.Output, + } + if err := setJSONAttr(span, "braintrust.input_json", classifierInput); err != nil { + return nil, err + } + + classifications, err := classifier.Run(ctx, taskResult) + if err != nil { + werr := fmt.Errorf("%w: classifier %q failed: %w", errClassifier, spanName, err) + recordSpanError(span, werr) + return nil, werr + } + + // Normalize and validate each classification: + // - default missing/empty Name to the classifier's resolved span name + // - require a non-empty ID (spec: each item must be a non-empty object) + items := make([]classificationItem, 0, len(classifications)) + outputByName := make(map[string][]map[string]any) + for _, c := range classifications { + if c.ID == "" { + werr := fmt.Errorf("%w: classifier %q returned a classification with empty ID", errClassifier, spanName) + recordSpanError(span, werr) + return nil, werr + } + name := c.Name + if name == "" { + name = spanName + } + payload := map[string]any{"id": c.ID} + if c.Label != "" { + payload["label"] = c.Label + } + if len(c.Metadata) > 0 { + payload["metadata"] = c.Metadata + } + items = append(items, classificationItem{name: name, payload: payload}) + outputByName[name] = append(outputByName[name], payload) + } + + // Log the classifier's output: always grouped by name as a slice of + // wire-format items, matching the spec's ClassificationItem shape. + if len(outputByName) > 0 { + if err := setJSONAttr(span, "braintrust.output_json", outputByName); err != nil { + return nil, err + } + } + + return items, nil +} + // permalink generates a URL to view the eval in Braintrust UI. func (e *eval[I, R]) permalink() string { appURL := e.session.AppPublicURL() @@ -641,6 +826,9 @@ func run[I, R any](ctx context.Context, opts Opts[I, R], s *auth.Session, tp *tr if opts.Task == nil { return nil, fmt.Errorf("%w: Task is required", errEval) } + if len(opts.Scorers) == 0 && len(opts.Classifiers) == 0 { + return nil, fmt.Errorf("%w: at least one of Scorers or Classifiers is required", errEval) + } // Create eval executor e, err := newEvalOpts(ctx, s, tp, apiClient, opts, project) @@ -681,6 +869,8 @@ func recordSpanError(span oteltrace.Span, err error) { switch { case errors.Is(err, errScorer): errType = "ErrScorer" + case errors.Is(err, errClassifier): + errType = "ErrClassifier" case errors.Is(err, errTaskRun): errType = "ErrTaskRun" case errors.Is(err, errCaseIterator): @@ -736,6 +926,7 @@ func testNewEval[I, R any]( dataset Dataset[I, R], task TaskFunc[I, R], scorers []Scorer[I, R], + classifiers []Classifier[I, R], parallelism int, ) *eval[I, R] { // Call low-level newEval with quiet=true for tests @@ -749,6 +940,7 @@ func testNewEval[I, R any]( dataset, task, scorers, + classifiers, parallelism, true, // quiet=true for tests ) diff --git a/eval/eval_integration_test.go b/eval/eval_integration_test.go index 9637edaf..27092280 100644 --- a/eval/eval_integration_test.go +++ b/eval/eval_integration_test.go @@ -719,6 +719,11 @@ func TestEval_NoProjectName(t *testing.T) { Task: T(func(ctx context.Context, input string) (string, error) { return input, nil }), + Scorers: []Scorer[string, string]{ + NewScorer("noop", func(ctx context.Context, _ TaskResult[string, string]) (Scores, error) { + return S(1.0), nil + }), + }, Quiet: true, }) diff --git a/eval/eval_test.go b/eval/eval_test.go index 4f85a465..f27d27d2 100644 --- a/eval/eval_test.go +++ b/eval/eval_test.go @@ -37,6 +37,14 @@ type unitTestEval[I, R any] struct { // It generates its own fake session, config, tracer, experiment/project IDs, etc. func newUnitTestEval[I, R any](t *testing.T, dataset Dataset[I, R], task TaskFunc[I, R], scorers []Scorer[I, R], parallelism int) *unitTestEval[I, R] { t.Helper() + return newUnitTestEvalWithClassifiers(t, dataset, task, scorers, nil, parallelism) +} + +// newUnitTestEvalWithClassifiers is the underlying constructor that accepts both +// scorers and classifiers. Most existing tests only exercise scorers and go +// through newUnitTestEval. +func newUnitTestEvalWithClassifiers[I, R any](t *testing.T, dataset Dataset[I, R], task TaskFunc[I, R], scorers []Scorer[I, R], classifiers []Classifier[I, R], parallelism int) *unitTestEval[I, R] { + t.Helper() // Create test tracer and exporter using oteltest tp, exporter := oteltest.Setup(t) @@ -56,6 +64,7 @@ func newUnitTestEval[I, R any](t *testing.T, dataset Dataset[I, R], task TaskFun dataset, task, scorers, + classifiers, parallelism, ) @@ -281,6 +290,7 @@ func TestPermalink_EscapesOrgName(t *testing.T) { NewDataset([]Case[testInput, testOutput]{}), task, nil, + nil, 1, ) diff --git a/examples/classifiers/classifiers.go b/examples/classifiers/classifiers.go new file mode 100644 index 00000000..db94532d --- /dev/null +++ b/examples/classifiers/classifiers.go @@ -0,0 +1,112 @@ +// This example shows how to use classifiers to categorize task outputs +// alongside (or instead of) numeric scorers. + +package main + +import ( + "context" + "log" + "regexp" + "strings" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/trace" + + "github.com/braintrustdata/braintrust-sdk-go" + "github.com/braintrustdata/braintrust-sdk-go/eval" +) + +func main() { + log.Println("🏷 Classifiers Example") + + tp := trace.NewTracerProvider() + defer tp.Shutdown(context.Background()) //nolint:errcheck + otel.SetTracerProvider(tp) + + bt, err := braintrust.New(tp, + braintrust.WithProject("go-sdk-examples"), + braintrust.WithBlockingLogin(true), + ) + if err != nil { + log.Fatalf("❌ Failed to initialize Braintrust: %v", err) + } + + evaluator := braintrust.NewEvaluator[string, string](bt) + + // Single-label classifier: returns one category per input. + intent := eval.NewClassifier("intent", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + switch { + case regexp.MustCompile(`(?i)thank`).MatchString(r.Input): + return eval.Classifications{{ID: "praise", Label: "Praise"}}, nil + case regexp.MustCompile(`(?i)password|reset`).MatchString(r.Input): + return eval.Classifications{{ID: "how_to", Label: "How To"}}, nil + case regexp.MustCompile(`(?i)damaged|refund`).MatchString(r.Input): + return eval.Classifications{{ID: "complaint", Label: "Complaint"}}, nil + default: + return eval.Classifications{{ID: "other"}}, nil + } + }) + + // Multi-label classifier: returns several classifications per input, + // all under the same name. The platform groups them together. + tone := eval.NewClassifier("tone", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + var out eval.Classifications + if strings.Contains(strings.ToLower(r.Input), "immediately") { + out = append(out, eval.Classification{ID: "urgent", Label: "Urgent"}) + } + if strings.Contains(strings.ToLower(r.Input), "please") { + out = append(out, eval.Classification{ID: "polite", Label: "Polite"}) + } + return out, nil + }) + + // Classifier with metadata: enriches each classification with structured + // context (word count, etc.). + responseQuality := eval.NewClassifier("response_quality", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + words := len(strings.Fields(r.Output)) + var id string + switch { + case strings.TrimSpace(r.Output) == "": + id = "no_response" + case words < 5: + id = "too_short" + default: + id = "informational" + } + return eval.Classifications{{ + ID: id, + Label: strings.Title(strings.ReplaceAll(id, "_", " ")), //nolint:staticcheck + Metadata: map[string]any{ + "word_count": words, + }, + }}, nil + }) + + // Classifiers can run alongside scorers in the same eval. + exactMatch := eval.NewScorer("exact_match", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Scores, error) { + if r.Output == r.Expected { + return eval.S(1.0), nil + } + return eval.S(0.0), nil + }) + + log.Println("🚀 Running evaluation...") + _, err = evaluator.Run(context.Background(), eval.Opts[string, string]{ + Experiment: "go-sdk-examples-classifiers", + Dataset: eval.NewDataset([]eval.Case[string, string]{ + {Input: "Thanks for the great support!", Expected: "praise"}, + {Input: "Please reset my password immediately.", Expected: "how_to"}, + {Input: "My order arrived damaged, I want a refund.", Expected: "complaint"}, + }), + Task: eval.T(func(_ context.Context, input string) (string, error) { + return "Thank you for reaching out. We'll look into it shortly.", nil + }), + Scorers: []eval.Scorer[string, string]{exactMatch}, + Classifiers: []eval.Classifier[string, string]{intent, tone, responseQuality}, + }) + if err != nil { + log.Printf("⚠️ Eval completed with errors: %v", err) + } else { + log.Println("✅ Eval completed successfully") + } +} diff --git a/examples/internal/classifiers/main.go b/examples/internal/classifiers/main.go new file mode 100644 index 00000000..151d4218 --- /dev/null +++ b/examples/internal/classifiers/main.go @@ -0,0 +1,123 @@ +// Kitchen-sink example for classifiers. Exercises every supported pattern +// so we can validate the feature end-to-end against a real workspace. + +package main + +import ( + "context" + "errors" + "log" + "strings" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/trace" + + "github.com/braintrustdata/braintrust-sdk-go" + "github.com/braintrustdata/braintrust-sdk-go/eval" +) + +func main() { + tp := trace.NewTracerProvider() + defer tp.Shutdown(context.Background()) //nolint:errcheck + otel.SetTracerProvider(tp) + + bt, err := braintrust.New(tp, + braintrust.WithProject("go-sdk-examples"), + braintrust.WithBlockingLogin(true), + ) + if err != nil { + log.Fatalf("init braintrust: %v", err) + } + + evaluator := braintrust.NewEvaluator[string, string](bt) + + // 1. Single-label classifier. Empty Name on the Classification + // defaults to the classifier's name ("category"). + singleLabel := eval.NewClassifier("category", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + if strings.Contains(strings.ToLower(r.Input), "hello") { + return eval.Classifications{{ID: "greeting"}}, nil + } + return eval.Classifications{{ID: "other"}}, nil + }) + + // 2. Multi-label classification: several items under the same name. + multiLabel := eval.NewClassifier("tone", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + var out eval.Classifications + lower := strings.ToLower(r.Input) + if strings.Contains(lower, "please") { + out = append(out, eval.Classification{ID: "polite", Label: "Polite"}) + } + if strings.Contains(lower, "immediately") || strings.Contains(lower, "now") { + out = append(out, eval.Classification{ID: "urgent", Label: "Urgent"}) + } + if len(out) == 0 { + out = append(out, eval.Classification{ID: "neutral", Label: "Neutral"}) + } + return out, nil + }) + + // 3. Classifier returning no classifications for some inputs. + emptyAllowed := eval.NewClassifier("flag", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + if strings.Contains(strings.ToLower(r.Output), "error") { + return eval.Classifications{{ID: "contains_error"}}, nil + } + return nil, nil + }) + + // 4. Classifier with rich metadata payloads. + withMetadata := eval.NewClassifier("length_bucket", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Classifications, error) { + words := len(strings.Fields(r.Output)) + bucket := "short" + switch { + case words > 50: + bucket = "long" + case words > 10: + bucket = "medium" + } + return eval.Classifications{{ + ID: bucket, + Label: strings.Title(bucket), //nolint:staticcheck + Metadata: map[string]any{ + "words": words, + "characters": len(r.Output), + }, + }}, nil + }) + + // 5. A classifier that fails — confirms non-fatal error handling and + // that classifier_errors lands in the eval span's metadata. + broken := eval.NewClassifier("broken", func(_ context.Context, _ eval.TaskResult[string, string]) (eval.Classifications, error) { + return nil, errors.New("intentional failure") + }) + + // 6. A regular scorer running in parallel with the classifiers, to + // confirm both passes happen concurrently per case. + exactMatch := eval.NewScorer("exact_match", func(_ context.Context, r eval.TaskResult[string, string]) (eval.Scores, error) { + if r.Output == r.Expected { + return eval.S(1.0), nil + } + return eval.S(0.0), nil + }) + + log.Println("Running classifiers kitchen-sink eval...") + result, err := evaluator.Run(context.Background(), eval.Opts[string, string]{ + Experiment: "go-sdk-internal-classifiers", + Dataset: eval.NewDataset([]eval.Case[string, string]{ + {Input: "Hello there", Expected: "Hi"}, + {Input: "Please reply immediately.", Expected: "Sure"}, + {Input: "trigger an error message", Expected: "ok"}, + }), + Task: eval.T(func(_ context.Context, input string) (string, error) { + return "Acknowledged: " + input, nil + }), + Scorers: []eval.Scorer[string, string]{exactMatch}, + Classifiers: []eval.Classifier[string, string]{singleLabel, multiLabel, emptyAllowed, withMetadata, broken}, + Parallelism: 2, + }) + if err != nil { + log.Printf("eval finished with errors (expected from 'broken'): %v", err) + } + if result != nil { + log.Printf("experiment: %s", result.Name()) + } +}