Skip to content

Commit 026c090

Browse files
realarkrealark
authored
Merge pull request #4 from braintrustdata/ark/BRA-3165-prompt-fetching-api-outline
Prompt Fetching
2 parents 3e3afe5 + 7eab143 commit 026c090

19 files changed

Lines changed: 1453 additions & 49 deletions

‎README.md‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ dependencies {
3131
### Evals
3232

3333
```java
34-
var config = BraintrustConfig.fromEnvironment();
35-
var openTelemetry = BraintrustTracing.of(config, true);
34+
var braintrust = Braintrust.get();
35+
var openTelemetry = braintrust.openTelemetryCreate();
3636
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
3737

3838
Function<String, String> getFoodType =
@@ -49,11 +49,8 @@ Function<String, String> getFoodType =
4949
return response.choices().get(0).message().content().orElse("").toLowerCase();
5050
};
5151

52-
var eval =
53-
Eval.<String, String>builder()
52+
var eval = braintrust.<String, String>evalBuilder()
5453
.name("java-eval-x-" + System.currentTimeMillis())
55-
.tracer(BraintrustTracing.getTracer(openTelemetry))
56-
.config(config)
5754
.cases(
5855
EvalCase.of("asparagus", "vegetable"),
5956
EvalCase.of("banana", "fruit"))
@@ -73,8 +70,8 @@ System.out.println("\n\n" + result.createReportString());
7370
### OpenAI Tracing
7471

7572
```java
76-
var braintrustConfig = BraintrustConfig.fromEnvironment();
77-
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
73+
var braintrust = Braintrust.get();
74+
var openTelemetry = braintrust.openTelemetryCreate();
7875
OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
7976

8077
var request =

‎build.gradle‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ dependencies {
6666
implementation 'org.apache.commons:commons-lang3:3.14.0'
6767
implementation 'com.google.code.findbugs:jsr305:3.0.2' // for @Nullable annotations
6868

69+
implementation "com.github.spullara.mustache.java:compiler:0.9.14"
70+
6971
testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}"
7072
testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}"
7173
testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}"
@@ -219,7 +221,7 @@ jar {
219221
'Implementation-Title': 'Braintrust Java SDK',
220222
'Implementation-Version': version,
221223
'Implementation-Vendor': 'Braintrust',
222-
'Main-Class': 'dev.braintrust.trace.SDKMain'
224+
'Main-Class': 'dev.braintrust.SDKMain'
223225
)
224226
}
225227
}

‎examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@
44
import com.anthropic.client.okhttp.AnthropicOkHttpClient;
55
import com.anthropic.models.messages.MessageCreateParams;
66
import com.anthropic.models.messages.Model;
7-
import dev.braintrust.config.BraintrustConfig;
7+
import dev.braintrust.Braintrust;
88
import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic;
9-
import dev.braintrust.trace.BraintrustTracing;
109

1110
/** Basic OTel + Anthropic instrumentation example */
1211
public class AnthropicInstrumentationExample {
@@ -18,9 +17,9 @@ public static void main(String[] args) throws Exception {
1817
+ " fail.\n");
1918
}
2019

21-
var braintrustConfig = BraintrustConfig.fromEnvironment();
22-
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
23-
var tracer = BraintrustTracing.getTracer(openTelemetry);
20+
var braintrust = Braintrust.get();
21+
var openTelemetry = braintrust.openTelemetryCreate();
22+
var tracer = openTelemetry.getTracer("my-instrumentation");
2423

2524
// Wrap Anthropic client with Braintrust instrumentation
2625
AnthropicClient anthropicClient =
@@ -35,7 +34,7 @@ public static void main(String[] args) throws Exception {
3534
}
3635

3736
var url =
38-
braintrustConfig.fetchProjectURI()
37+
braintrust.projectUri()
3938
+ "/logs?r=%s&s=%s"
4039
.formatted(
4140
rootSpan.getSpanContext().getTraceId(),

‎examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package dev.braintrust.examples;
22

3-
import dev.braintrust.config.BraintrustConfig;
4-
import dev.braintrust.trace.BraintrustTracing;
3+
import dev.braintrust.Braintrust;
54
import io.opentelemetry.api.GlobalOpenTelemetry;
65
import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter;
76
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
@@ -51,8 +50,8 @@ public static void main(String[] args) throws Exception {
5150
// NOTE: there are many ways to set up otel builders, etc.
5251
// The important line is here: call enable with your otel builders and braintrust will
5352
// export open telemetry data in addition to your existing setup
54-
var braintrustConfig = BraintrustConfig.fromEnvironment();
55-
BraintrustTracing.enable(braintrustConfig, tracerBuilder, loggerBuilder, meterBuilder);
53+
var braintrust = Braintrust.get();
54+
braintrust.openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder);
5655

5756
var openTelemetry =
5857
OpenTelemetrySdk.builder()
@@ -74,7 +73,7 @@ public static void main(String[] args) throws Exception {
7473
span.end();
7574
}
7675
var url =
77-
braintrustConfig.fetchProjectURI()
76+
braintrust.projectUri()
7877
+ "/logs?r=%s&s=%s"
7978
.formatted(
8079
span.getSpanContext().getTraceId(),

‎examples/src/main/java/dev/braintrust/examples/ExperimentExample.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,16 @@
33
import com.openai.client.okhttp.OpenAIOkHttpClient;
44
import com.openai.models.ChatModel;
55
import com.openai.models.chat.completions.ChatCompletionCreateParams;
6-
import dev.braintrust.config.BraintrustConfig;
7-
import dev.braintrust.eval.Eval;
6+
import dev.braintrust.Braintrust;
87
import dev.braintrust.eval.EvalCase;
98
import dev.braintrust.eval.Scorer;
109
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
11-
import dev.braintrust.trace.BraintrustTracing;
1210
import java.util.function.Function;
1311

1412
public class ExperimentExample {
1513
public static void main(String[] args) throws Exception {
16-
var config = BraintrustConfig.fromEnvironment();
17-
var openTelemetry = BraintrustTracing.of(config, true);
14+
var braintrust = Braintrust.get();
15+
var openTelemetry = braintrust.openTelemetryCreate();
1816
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
1917

2018
Function<String, String> getFoodType =
@@ -32,13 +30,12 @@ public static void main(String[] args) throws Exception {
3230
};
3331

3432
var eval =
35-
Eval.<String, String>builder()
33+
braintrust
34+
.<String, String>evalBuilder()
3635
.name("java-eval-x-" + System.currentTimeMillis()) // NOTE: if you use a
3736
// constant, additional runs
3837
// will append new cases to
3938
// the same experiment
40-
.tracer(BraintrustTracing.getTracer(openTelemetry))
41-
.config(config)
4239
.cases(
4340
EvalCase.of("strawberry", "fruit"),
4441
EvalCase.of("asparagus", "vegetable"),

‎examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,31 +5,34 @@
55
import com.openai.models.ChatModel;
66
import com.openai.models.chat.completions.ChatCompletionCreateParams;
77
import com.openai.models.chat.completions.ChatCompletionStreamOptions;
8-
import dev.braintrust.config.BraintrustConfig;
8+
import dev.braintrust.Braintrust;
99
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
10-
import dev.braintrust.trace.BraintrustTracing;
1110

1211
/** Basic OTel + OpenAI instrumentation example */
1312
public class OpenAIInstrumentationExample {
13+
1414
public static void main(String[] args) throws Exception {
1515
if (null == System.getenv("OPENAI_API_KEY")) {
1616
System.err.println(
1717
"\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n");
1818
}
19-
var braintrustConfig = BraintrustConfig.fromEnvironment();
20-
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
21-
var tracer = BraintrustTracing.getTracer(openTelemetry);
19+
var braintrust = Braintrust.get();
20+
var openTelemetry = braintrust.openTelemetryCreate();
2221
OpenAIClient openAIClient =
2322
BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
24-
var rootSpan = tracer.spanBuilder("openai-java-instrumentation-example").startSpan();
23+
var rootSpan =
24+
openTelemetry
25+
.getTracer("my-instrumentation")
26+
.spanBuilder("openai-java-instrumentation-example")
27+
.startSpan();
2528
try (var ignored = rootSpan.makeCurrent()) {
2629
chatCompletionsExample(openAIClient);
2730
// chatCompletionsStreamingExample(openAIClient);
2831
} finally {
2932
rootSpan.end();
3033
}
3134
var url =
32-
braintrustConfig.fetchProjectURI()
35+
braintrust.projectUri()
3336
+ "/logs?r=%s&s=%s"
3437
.formatted(
3538
rootSpan.getSpanContext().getTraceId(),

‎examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package dev.braintrust.examples;
2+
3+
import static dev.braintrust.instrumentation.openai.BraintrustOpenAI.buildChatCompletionsPrompt;
4+
import static dev.braintrust.instrumentation.openai.BraintrustOpenAI.wrapOpenAI;
5+
6+
import com.openai.client.OpenAIClient;
7+
import com.openai.client.okhttp.OpenAIOkHttpClient;
8+
import dev.braintrust.Braintrust;
9+
import dev.braintrust.prompt.BraintrustPromptLoader.PromptLoadRequest;
10+
import java.util.Map;
11+
12+
public class PromptFetchingExample {
13+
public static void main(String... args) {
14+
if (null == System.getenv("OPENAI_API_KEY")) {
15+
System.err.println(
16+
"\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n");
17+
}
18+
var braintrust = Braintrust.get();
19+
var openTelemetry = braintrust.openTelemetryCreate();
20+
21+
OpenAIClient openAIClient = wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
22+
23+
{ // simple example
24+
var prompt = braintrust.promptLoader().load("kind-greeter-69d2");
25+
var response =
26+
openAIClient
27+
.chat()
28+
.completions()
29+
.create(
30+
buildChatCompletionsPrompt(
31+
prompt, Map.of("name", "Sam Malone")));
32+
System.out.println("got response: %s".formatted(response.choices().get(0).message()));
33+
}
34+
{ // more complex prompt options
35+
var prompt =
36+
braintrust
37+
.promptLoader()
38+
.load(
39+
PromptLoadRequest.builder()
40+
.projectName("andrew-misc")
41+
.promptSlug("unkind-greeter-fd4c")
42+
.version("cbbc711da9f7d445")
43+
.defaults("model", "gpt-3.5-turbo")
44+
.build());
45+
var response =
46+
openAIClient
47+
.chat()
48+
.completions()
49+
.create(
50+
buildChatCompletionsPrompt(
51+
prompt, Map.of("name", "Frasier Crane")));
52+
System.out.println("got response: %s".formatted(response));
53+
}
54+
55+
System.out.println(
56+
"\n\n Example complete! View your data in Braintrust: %s\n"
57+
.formatted(braintrust.projectUri() + "/logs"));
58+
}
59+
}

‎examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
package dev.braintrust.examples;
22

3-
import dev.braintrust.config.BraintrustConfig;
4-
import dev.braintrust.trace.BraintrustTracing;
3+
import dev.braintrust.Braintrust;
54

65
public class SimpleOpenTelemetryExample {
76
public static void main(String[] args) throws Exception {
8-
var braintrustConfig = BraintrustConfig.fromEnvironment();
9-
var openTelemetry = BraintrustTracing.quickstart();
10-
var tracer = BraintrustTracing.getTracer(openTelemetry);
7+
var braintrust = Braintrust.get();
8+
var openTelemetry = braintrust.openTelemetryCreate();
9+
var tracer = openTelemetry.getTracer("my-instrumentation");
1110

1211
var span = tracer.spanBuilder("hello-java").startSpan();
1312
try (var ignored = span.makeCurrent()) {
@@ -18,7 +17,7 @@ public static void main(String[] args) throws Exception {
1817
span.end();
1918
}
2019
var url =
21-
braintrustConfig.fetchProjectURI()
20+
braintrust.projectUri()
2221
+ "/logs?r=%s&s=%s"
2322
.formatted(
2423
span.getSpanContext().getTraceId(),

0 commit comments

Comments
 (0)