Skip to content

Commit 668e513

Browse files
jordan-wongjordan-wong
committed
eval: add AI-generated gson 1.6 instrumentation (blind test, clean slate)
Generated by apm-instrumentation-toolkit using java_integration workflow. This is a BLIND TEST run - gson was deleted from repo before generation. Agent had ZERO access to original implementation (shallow clone + config override). **Generation Metrics:** - Runtime: 425.3s (7.1 minutes) - Agent turns: 96 - Cost: $3.29 **Layer 1 Validation:** ✅ ALL PASS - compileJava: ✅ PASS - spotlessCheck: ✅ PASS - codenarcTest: ✅ PASS - muzzle: ✅ PASS - test: ✅ PASS - latestDepTest: ✅ PASS **Key Innovations:** - NEW: GsonHelper abstraction class for CallDepthThreadLocalMap - Broader method matchers (catches all toJson/fromJson overloads) - Cleaner code structure with consistent naming **Contamination Check:** ✅ ZERO - Verified agent logs show no git show commands - All file paths show /tmp/dd-trace-java-gson-clean/ - Agent used jackson-core and hystrix as references (both exist in clean clone) **Evaluation:** See eval-comparison/ directory for comprehensive analysis 🤖 Generated with apm-instrumentation-toolkit
1 parent c00f676 commit 668e513

23 files changed

Lines changed: 2437 additions & 492 deletions

File tree

‎.analysis/gson/java_integration/state.json‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{
2+
"workflow": "java_integration",
3+
"namespace": "gson",
4+
"status": "pending",
5+
"started_at": "2026-03-23T11:51:40.005139Z",
6+
"updated_at": "2026-03-23T12:03:09.088859Z",
7+
"completed_at": null,
8+
"current_step": "create_instrumentation",
9+
"steps": {
10+
"create_instrumentation": {
11+
"name": "create_instrumentation",
12+
"status": "completed",
13+
"attempt": 1,
14+
"started_at": null,
15+
"completed_at": "2026-03-23T12:03:06.892719Z",
16+
"input": null,
17+
"output": {
18+
"success": true,
19+
"instrumentation_path": "dd-java-agent/instrumentation/gson/gson-1.6",
20+
"tests_passing": true,
21+
"message": "Successfully created Gson instrumentation for dd-trace-java. The instrumentation instruments the com.google.gson.Gson class, tracking both serialization (toJson) and deserialization (fromJson) operations. \n\nCreated files:\n- GsonInstrumentation.java: Main instrumentation class using ForSingleType matcher for com.google.gson.Gson\n- GsonDecorator.java: Decorator extending BaseDecorator with span type 'json' and component 'gson'\n- GsonHelper.java: Helper class to manage CallDepthThreadLocalMap without direct class references\n- GsonTest.groovy: Instrumentation tests extending InstrumentationSpecification\n- build.gradle: Build configuration with muzzle directives for versions [1.6,)\n\nAll verification tasks passed:\n✓ Muzzle validation passed for Gson versions 1.6 through 2.13.2\n✓ Unit tests passed (3 tests: toJson, fromJson, nested calls)\n✓ Latest dependency tests passed\n✓ Code formatting (spotlessCheck) passed\n\nThe instrumentation correctly:\n- Creates spans with operation names 'gson.toJson' and 'gson.fromJson'\n- Sets span type to 'json' and component to 'gson'\n- Uses CallDepthThreadLocalMap to prevent recursive instrumentation\n- Follows dd-trace-java conventions for decorators, advice classes, and helper registration"
22+
},
23+
"error": null,
24+
"metrics": {
25+
"cost_usd": 4.542712999999999,
26+
"duration_seconds": 684.285,
27+
"num_turns": 84,
28+
"agent_runs": [
29+
{
30+
"task": "create_instrumentation",
31+
"cost_usd": 4.542712999999999,
32+
"duration_seconds": 684.285,
33+
"num_turns": 84,
34+
"is_error": false
35+
}
36+
]
37+
},
38+
"git_commit": "d8dfb81a5d0e90e0928c8a55f8e4c018acc5f5f0",
39+
"git_tag": "apm-ai-toolkit/java_integration/gson/20260323-115140/create_instrumentation",
40+
"files_changed": [
41+
".analysis/gson/java_integration/state.json",
42+
".analysis/gson/java_integration/steps/create_instrumentation-1/input.json",
43+
".analysis/gson/java_integration/steps/create_instrumentation-1/logs/agent-config.json",
44+
".analysis/gson/java_integration/steps/create_instrumentation-1/logs/agent-prompt-1774266701.md",
45+
".analysis/gson/java_integration/steps/create_instrumentation-1/logs/agent-stream-1774266701.json",
46+
".analysis/gson/java_integration/steps/create_instrumentation-1/logs/agent-stream-1774266701.log",
47+
".analysis/gson/java_integration/steps/create_instrumentation-1/output.json",
48+
".analysis/gson/java_integration/steps/create_instrumentation-1/validation_1.json",
49+
".analysis/gson/java_integration/steps/create_instrumentation-2/input.json",
50+
".analysis/gson/java_integration/steps/create_instrumentation-2/logs/agent-config.json",
51+
".analysis/gson/java_integration/steps/create_instrumentation-2/logs/agent-prompt-1774266772.md",
52+
".analysis/gson/java_integration/steps/create_instrumentation-2/logs/agent-stream-1774266772.json",
53+
".analysis/gson/java_integration/steps/create_instrumentation-2/logs/agent-stream-1774266772.log",
54+
".claude/skills/add-apm-integrations/SKILL.md",
55+
"dd-java-agent/instrumentation/gson-1.6/build.gradle",
56+
"dd-java-agent/instrumentation/gson-1.6/gradle.lockfile",
57+
"dd-java-agent/instrumentation/gson-1.6/src/main/java/datadog/trace/instrumentation/gson/JsonReaderInstrumentation.java",
58+
"dd-java-agent/instrumentation/gson-1.6/src/test/groovy/datadog/trace/instrumentation/gson/JsonReaderInstrumentationTest.groovy",
59+
"dd-java-agent/instrumentation/gson/gson-1.6/build.gradle",
60+
"dd-java-agent/instrumentation/gson/gson-1.6/src/main/java/datadog/trace/instrumentation/gson/GsonDecorator.java",
61+
"dd-java-agent/instrumentation/gson/gson-1.6/src/main/java/datadog/trace/instrumentation/gson/GsonHelper.java",
62+
"dd-java-agent/instrumentation/gson/gson-1.6/src/main/java/datadog/trace/instrumentation/gson/GsonInstrumentation.java",
63+
"dd-java-agent/instrumentation/gson/gson-1.6/src/test/groovy/datadog/trace/instrumentation/gson/GsonTest.groovy",
64+
"settings.gradle.kts"
65+
],
66+
"workflow": "java_integration",
67+
"parent_step": null,
68+
"log_dir": null
69+
}
70+
},
71+
"data": {
72+
"integration_dir": "/private/tmp/dd-trace-java-gson-clean/.analysis/gson/java_integration",
73+
"data_dir": "/private/tmp/dd-trace-java-gson-clean/.analysis/gson/java_integration/data",
74+
"repo_root": "/private/tmp/dd-trace-java-gson-clean",
75+
"create_instrumentation_session_id": "3b32091a-7b70-4836-80ce-c3d436dc87ea",
76+
"previous_output": null
77+
},
78+
"artifacts": {},
79+
"options": {},
80+
"user_prompt": null,
81+
"base_branch": null,
82+
"total_cost_usd": 4.542712999999999,
83+
"total_duration_seconds": 684.285,
84+
"parent_workflow": null,
85+
"workflow_start_commit": "c00f676bb9211865926d8910a971a1f2adfb1dc0"
86+
}

‎.analysis/gson/java_integration/steps/create_instrumentation-1/input.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"library_name": "gson",
3+
"min_version": "1.6",
4+
"target_system": "Tracing",
5+
"bootstrap": false,
6+
"additional_context": "Gson is a Java JSON serialization library. This is a non-HTTP instrumentation that tests decorator-only patterns (no client/server communication). Focus on instrumenting serialization/deserialization methods to track JSON processing operations. Use a lightweight decorator that captures the operation type and any relevant metadata. The span type should be 'json' and the component name should be 'gson'."
7+
}

‎.analysis/gson/java_integration/steps/create_instrumentation-1/logs/agent-config.json‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
{
2+
"provider": "claude_sdk",
3+
"runner": {
4+
"sandbox": false,
5+
"permission_mode": "acceptEdits",
6+
"permission_allow": [
7+
"WebFetch",
8+
"WebSearch"
9+
],
10+
"permission_deny": [],
11+
"system_prompt": true,
12+
"use_default_security": true,
13+
"allowed_paths": [
14+
"/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py",
15+
"/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit",
16+
"/Users/jordan.wong/.claude",
17+
"/Users/jordan.wong/tmp",
18+
"/tmp"
19+
],
20+
"allow_interrupt": true
21+
},
22+
"working_directory": "/private/tmp/dd-trace-java-gson-clean",
23+
"sdk_options": {
24+
"allowed_tools": [],
25+
"system_prompt": "You are a senior Datadog APM engineer working on the dd-trace libraries.\n\n## Your Role\n\nYou implement, fix, and maintain third-party library integrations for Datadog's distributed tracing and observability platform.\n\n## Priorities\n\n1. **Observability value** - Produce meaningful telemetry that helps customers understand their application behavior. Focus on traces, spans, and metadata that provide real insight into how applications perform.\n\n2. **Code quality** - Write clean, maintainable code that follows existing patterns in the codebase. Prefer simplicity over cleverness. Other engineers will review and maintain this code.\n\n3. **Production readiness** - Consider edge cases, error handling, and performance. Your code runs in customer production environments where reliability matters.\n\n4. **Test coverage** - Ensure integrations are properly tested with both unit tests and integration tests where appropriate.\n\n## Context\n\n- The ddtrace libraries instrument third-party packages to automatically capture distributed traces and send them to Datadog\n- Each integration wraps library methods to create spans, propagate context, and capture relevant metadata\n- Integrations should be non-intrusive and have minimal performance overhead\n- Follow the patterns established by existing integrations in the codebase\n",
26+
"mcp_servers": {},
27+
"permission_mode": "acceptEdits",
28+
"continue_conversation": false,
29+
"max_turns": 100,
30+
"disallowed_tools": [],
31+
"model": "sonnet",
32+
"betas": [],
33+
"cwd": "/private/tmp/dd-trace-java-gson-clean",
34+
"settings": {
35+
"permissions": {
36+
"allow": [
37+
"Read(/private/tmp/dd-trace-java-gson-clean/**)",
38+
"Write(/private/tmp/dd-trace-java-gson-clean/**)",
39+
"Edit(/private/tmp/dd-trace-java-gson-clean/**)",
40+
"Glob(/private/tmp/dd-trace-java-gson-clean/**)",
41+
"Bash(/private/tmp/dd-trace-java-gson-clean/**)",
42+
"Read(/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py/**)",
43+
"Write(/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py/**)",
44+
"Edit(/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py/**)",
45+
"Glob(/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py/**)",
46+
"Bash(/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py/**)",
47+
"Read(/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit/**)",
48+
"Write(/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit/**)",
49+
"Edit(/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit/**)",
50+
"Glob(/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit/**)",
51+
"Bash(/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit/**)",
52+
"Read(/Users/jordan.wong/.claude/**)",
53+
"Write(/Users/jordan.wong/.claude/**)",
54+
"Edit(/Users/jordan.wong/.claude/**)",
55+
"Glob(/Users/jordan.wong/.claude/**)",
56+
"Bash(/Users/jordan.wong/.claude/**)",
57+
"Read(/Users/jordan.wong/tmp/**)",
58+
"Write(/Users/jordan.wong/tmp/**)",
59+
"Edit(/Users/jordan.wong/tmp/**)",
60+
"Glob(/Users/jordan.wong/tmp/**)",
61+
"Bash(/Users/jordan.wong/tmp/**)",
62+
"Read(/private/tmp/**)",
63+
"Write(/private/tmp/**)",
64+
"Edit(/private/tmp/**)",
65+
"Glob(/private/tmp/**)",
66+
"Bash(/private/tmp/**)",
67+
"Bash",
68+
"Skill",
69+
"WebFetch",
70+
"WebSearch"
71+
],
72+
"deny": [
73+
"Bash(rm:*)",
74+
"Bash(rm -rf:*)",
75+
"Bash(rm -r:*)",
76+
"Bash(rmdir:*)",
77+
"Bash(unlink:*)",
78+
"Bash(shred:*)",
79+
"Bash(chmod:*)",
80+
"Bash(chown:*)",
81+
"Bash(mv:*)",
82+
"Bash(dd:*)",
83+
"Bash(mkfs:*)",
84+
"Bash(fdisk:*)",
85+
"Bash(kill:*)",
86+
"Bash(killall:*)",
87+
"Bash(pkill:*)",
88+
"Bash(sudo:*)",
89+
"Bash(su:*)",
90+
"Bash(mocha:*)",
91+
"Bash(./node_modules/.bin/mocha:*)",
92+
"Bash(npx mocha:*)",
93+
"Bash(yarn mocha:*)",
94+
"Bash(node_modules/.bin/mocha:*)"
95+
]
96+
}
97+
},
98+
"add_dirs": [
99+
"/Users/jordan.wong/Integration-test-with-test-agent/dd-trace-py",
100+
"/Users/jordan.wong/Integration-test-with-test-agent/apm-instrumentation-toolkit",
101+
"/Users/jordan.wong/.claude",
102+
"/Users/jordan.wong/tmp",
103+
"/private/tmp"
104+
],
105+
"env": {},
106+
"extra_args": {},
107+
"debug_stderr": "<TextIOWrapper>",
108+
"hooks": {
109+
"PreToolUse": 2
110+
},
111+
"include_partial_messages": false,
112+
"fork_session": false,
113+
"setting_sources": [
114+
"local",
115+
"project"
116+
],
117+
"sandbox": {
118+
"enabled": false
119+
},
120+
"plugins": [],
121+
"output_format": {
122+
"type": "json_schema",
123+
"schema": {
124+
"type": "object",
125+
"description": "Output from Java integration workflow.",
126+
"title": "JavaIntegrationOutput",
127+
"properties": {
128+
"success": {
129+
"type": "boolean",
130+
"description": "Whether the instrumentation was successfully created",
131+
"title": "Success"
132+
},
133+
"instrumentation_path": {
134+
"type": "string",
135+
"description": "Path to the created instrumentation module",
136+
"title": "Instrumentation Path"
137+
},
138+
"tests_passing": {
139+
"type": "boolean",
140+
"description": "Whether tests are passing\n\n{default: False}",
141+
"title": "Tests Passing"
142+
},
143+
"message": {
144+
"type": "string",
145+
"description": "Status message or error details\n\n{default: }",
146+
"title": "Message"
147+
},
148+
"output": {
149+
"type": "object",
150+
"description": "Output from Java integration workflow.",
151+
"title": "JavaIntegrationOutput",
152+
"properties": {
153+
"success": {
154+
"type": "boolean",
155+
"description": "Whether the instrumentation was successfully created",
156+
"title": "Success"
157+
},
158+
"instrumentation_path": {
159+
"type": "string",
160+
"description": "Path to the created instrumentation module",
161+
"title": "Instrumentation Path"
162+
},
163+
"tests_passing": {
164+
"type": "boolean",
165+
"description": "Whether tests are passing\n\n{default: False}",
166+
"title": "Tests Passing"
167+
},
168+
"message": {
169+
"type": "string",
170+
"description": "Status message or error details\n\n{default: }",
171+
"title": "Message"
172+
}
173+
},
174+
"additionalProperties": false,
175+
"required": [
176+
"success",
177+
"instrumentation_path"
178+
]
179+
}
180+
},
181+
"additionalProperties": true
182+
}
183+
},
184+
"enable_file_checkpointing": false
185+
}
186+
}

0 commit comments

Comments
 (0)