Skip to content

Commit 7384fe2

Browse files
anandgupta42claude
andcommitted
feat: e2e tests, performance benchmarks, and UX gap fixes
## E2E Integration Tests (41 tests) - `feature-discovery-e2e.test.ts` (29 tests): Full warehouse-add → suggestions flow, progressive disclosure chain, plan refinement session, telemetry event validation with mocked Dispatcher - `performance-regression.test.ts` (12 tests): 1000x suggestion generation < 50ms, 10000x progressive lookup < 50ms, 100k phrase detection < 200ms, output determinism verification ## UX Gap Fixes - **Suggestion deduplication**: Progressive hints shown at most once per session per tool via `shownProgressiveSuggestions` Set. Running `sql_execute` 10 times no longer repeats the same tip. - **Approval false positives**: "yes, but change X" now correctly classified as "refine" not "approve". Added `refinementQualifiers` (" but ", " however ", " except ", " change ", etc.) that override approval detection. "no" uses `\bno\b` word boundary to avoid matching "know", "notion", etc. - **Revision cap communication**: When 5 revision cap is hit, a synthetic message informs the LLM to tell the user. Telemetry tracks "cap_reached". - **Warehouse add latency**: Suggestion gathering now uses `Promise.all` + `Promise.race` with 1500ms timeout. Slow schema/dbt checks silently skipped instead of blocking the response. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 57a2e78 commit 7384fe2

7 files changed

Lines changed: 976 additions & 49 deletions

File tree

packages/opencode/src/altimate/telemetry/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ export namespace Telemetry {
394394
timestamp: number
395395
session_id: string
396396
revision_number: number
397-
action: "refine" | "approve" | "reject"
397+
action: "refine" | "approve" | "reject" | "cap_reached"
398398
}
399399
// altimate_change end
400400
| {

packages/opencode/src/altimate/tools/post-connect-suggestions.ts

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
* After warehouse connect, users often don't know what to do next.
55
* This module provides contextual suggestions based on the user's
66
* environment and progressive next-step hints after tool usage.
7+
*
8+
* Deduplication: progressive suggestions are shown at most once per
9+
* session per tool to avoid repetitive hints.
710
*/
811

912
import { Telemetry } from "../../telemetry"
@@ -17,6 +20,17 @@ export namespace PostConnectSuggestions {
1720
toolsUsedInSession: string[]
1821
}
1922

23+
/**
24+
* Set of progressive suggestion keys already shown in this process.
25+
* Reset when the process restarts (per-session lifetime).
26+
*/
27+
const shownProgressiveSuggestions = new Set<string>()
28+
29+
/** Reset shown suggestions (useful for testing). */
30+
export function resetShownSuggestions(): void {
31+
shownProgressiveSuggestions.clear()
32+
}
33+
2034
export function getPostConnectSuggestions(ctx: SuggestionContext): string {
2135
const suggestions: string[] = []
2236

@@ -60,7 +74,8 @@ export namespace PostConnectSuggestions {
6074

6175
/**
6276
* Progressive disclosure: suggest next tool based on what was just used.
63-
* Returns null if no suggestion applies or tool is unknown.
77+
* Returns null if no suggestion applies, tool is unknown, or the
78+
* suggestion was already shown in this session (deduplication).
6479
*/
6580
export function getProgressiveSuggestion(
6681
lastToolUsed: string,
@@ -76,7 +91,17 @@ export namespace PostConnectSuggestions {
7691
"Schema indexed! You can now use sql_analyze for quality checks, schema_inspect for exploration, and lineage_check for data flow analysis.",
7792
warehouse_add: null, // Handled by post-connect suggestions
7893
}
79-
return progression[lastToolUsed] ?? null
94+
95+
const suggestion = progression[lastToolUsed] ?? null
96+
if (!suggestion) return null
97+
98+
// Deduplicate: only show each progressive suggestion once per session
99+
if (shownProgressiveSuggestions.has(lastToolUsed)) {
100+
return null
101+
}
102+
shownProgressiveSuggestions.add(lastToolUsed)
103+
104+
return suggestion
80105
}
81106

82107
/**

packages/opencode/src/altimate/tools/warehouse-add.ts

Lines changed: 43 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -44,41 +44,54 @@ IMPORTANT: For private key file paths, always use "private_key_path" (not "priva
4444
})
4545

4646
if (result.success) {
47-
// altimate_change start — append post-connect feature suggestions
47+
// altimate_change start — append post-connect feature suggestions (async, non-blocking)
4848
let output = `Successfully added warehouse '${result.name}' (type: ${result.type}).\n\nUse warehouse_test to verify connectivity.`
49+
50+
// Run suggestion gathering concurrently with a timeout to avoid
51+
// adding noticeable latency to the warehouse add response.
4952
try {
50-
const schemaCache = await Dispatcher.call("schema.cache_status", {}).catch(() => null)
51-
const schemaIndexed = (schemaCache?.total_tables ?? 0) > 0
52-
const warehouseList = await Dispatcher.call("warehouse.list", {}).catch(() => ({ warehouses: [] }))
53+
const SUGGESTION_TIMEOUT_MS = 1500
54+
const suggestionPromise = (async () => {
55+
const [schemaCache, warehouseList, dbtInfo] = await Promise.all([
56+
Dispatcher.call("schema.cache_status", {}).catch(() => null),
57+
Dispatcher.call("warehouse.list", {}).catch(() => ({ warehouses: [] })),
58+
import("./project-scan")
59+
.then((m) => m.detectDbtProject(process.cwd()))
60+
.catch(() => ({ found: false })),
61+
])
62+
const schemaIndexed = (schemaCache?.total_tables ?? 0) > 0
63+
const dbtDetected = dbtInfo.found
5364

54-
let dbtDetected = false
55-
try {
56-
const { detectDbtProject } = await import("./project-scan")
57-
const dbtInfo = await detectDbtProject(process.cwd())
58-
dbtDetected = dbtInfo.found
59-
} catch {
60-
// project-scan unavailable — skip dbt detection
61-
}
65+
const suggestionCtx: PostConnectSuggestions.SuggestionContext = {
66+
warehouseType: result.type,
67+
schemaIndexed,
68+
dbtDetected,
69+
connectionCount: warehouseList.warehouses.length,
70+
toolsUsedInSession: [],
71+
}
72+
return { suggestionCtx, schemaIndexed, dbtDetected }
73+
})()
6274

63-
const suggestionCtx: PostConnectSuggestions.SuggestionContext = {
64-
warehouseType: result.type,
65-
schemaIndexed,
66-
dbtDetected,
67-
connectionCount: warehouseList.warehouses.length,
68-
toolsUsedInSession: [],
69-
}
70-
output += PostConnectSuggestions.getPostConnectSuggestions(suggestionCtx)
75+
const timeoutPromise = new Promise<null>((resolve) =>
76+
setTimeout(() => resolve(null), SUGGESTION_TIMEOUT_MS),
77+
)
78+
const suggestionResult = await Promise.race([suggestionPromise, timeoutPromise])
7179

72-
// Derive suggestions list from the same context to avoid drift
73-
const suggestionsShown = ["sql_execute", "sql_analyze", "lineage_check", "schema_detect_pii"]
74-
if (!suggestionCtx.schemaIndexed) suggestionsShown.unshift("schema_index")
75-
if (suggestionCtx.dbtDetected) suggestionsShown.push("dbt-develop", "dbt-troubleshoot")
76-
if (suggestionCtx.connectionCount > 1) suggestionsShown.push("data_diff")
77-
PostConnectSuggestions.trackSuggestions({
78-
suggestionType: "post_warehouse_connect",
79-
suggestionsShown,
80-
warehouseType: result.type,
81-
})
80+
if (suggestionResult) {
81+
const { suggestionCtx } = suggestionResult
82+
output += PostConnectSuggestions.getPostConnectSuggestions(suggestionCtx)
83+
84+
// Derive suggestions list from the same context to avoid drift
85+
const suggestionsShown = ["sql_execute", "sql_analyze", "lineage_check", "schema_detect_pii"]
86+
if (!suggestionCtx.schemaIndexed) suggestionsShown.unshift("schema_index")
87+
if (suggestionCtx.dbtDetected) suggestionsShown.push("dbt-develop", "dbt-troubleshoot")
88+
if (suggestionCtx.connectionCount > 1) suggestionsShown.push("data_diff")
89+
PostConnectSuggestions.trackSuggestions({
90+
suggestionType: "post_warehouse_connect",
91+
suggestionsShown,
92+
warehouseType: result.type,
93+
})
94+
}
8295
} catch {
8396
// Suggestions must never break the add flow
8497
}

packages/opencode/src/session/prompt.ts

Lines changed: 55 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -628,26 +628,67 @@ export namespace SessionPrompt {
628628
planHasWritten = await Filesystem.exists(planPath)
629629
}
630630
// If plan was already written and user sent a new message, this is a refinement
631-
if (planHasWritten && step > 1 && planRevisionCount < 5) {
632-
planRevisionCount++
631+
if (planHasWritten && step > 1) {
633632
// Detect approval phrases in the last user message text
634633
const lastUserMsg = msgs.findLast((m) => m.info.role === "user")
635634
const userText = lastUserMsg?.parts
636635
.filter((p): p is MessageV2.TextPart => p.type === "text" && !("synthetic" in p && p.synthetic))
637636
.map((p) => p.text.toLowerCase())
638637
.join(" ") ?? ""
639-
const rejectionPhrases = ["no", "don't", "stop", "reject", "not good", "undo", "abort", "start over", "wrong"]
640-
const approvalPhrases = ["looks good", "proceed", "approved", "approve", "lgtm", "go ahead", "ship it", "yes", "perfect"]
641-
const isRejection = rejectionPhrases.some((phrase) => userText.includes(phrase))
642-
const isApproval = !isRejection && approvalPhrases.some((phrase) => userText.includes(phrase))
643-
const action = isRejection ? "reject" : isApproval ? "approve" : "refine"
644-
Telemetry.track({
645-
type: "plan_revision",
646-
timestamp: Date.now(),
647-
session_id: sessionID,
648-
revision_number: planRevisionCount,
649-
action,
650-
})
638+
639+
if (planRevisionCount >= 5) {
640+
// Cap reached — track and inject a synthetic hint so the LLM informs the user
641+
Telemetry.track({
642+
type: "plan_revision",
643+
timestamp: Date.now(),
644+
session_id: sessionID,
645+
revision_number: planRevisionCount,
646+
action: "cap_reached",
647+
})
648+
// Append a synthetic text part to the last user message in the local msgs copy
649+
// so the LLM sees the limit and can communicate it. This does not persist.
650+
if (lastUserMsg) {
651+
lastUserMsg.parts = [
652+
...lastUserMsg.parts,
653+
{
654+
type: "text" as const,
655+
id: PartID.ascending(),
656+
sessionID,
657+
messageID: lastUserMsg.info.id,
658+
text: "\n\n[System note: This plan has reached the maximum revision limit (5). Please inform the user and suggest finalizing the plan or starting a new planning session.]",
659+
synthetic: true,
660+
},
661+
]
662+
}
663+
} else {
664+
planRevisionCount++
665+
666+
// Refinement qualifiers: if the user says "yes, but ..." or "approve, however ..."
667+
// they intend to refine, not approve. Check for these before pure approval.
668+
const refinementQualifiers = [" but ", " however ", " except ", " change ", " modify ", " update ", " instead ", " although ", " with the following", " with these"]
669+
const hasRefinementQualifier = refinementQualifiers.some((q) => userText.includes(q))
670+
671+
const rejectionPhrases = ["don't", "stop", "reject", "not good", "undo", "abort", "start over", "wrong"]
672+
// "no" as a standalone word to avoid matching "know", "notion", etc.
673+
const rejectionWords = ["no"]
674+
const approvalPhrases = ["looks good", "proceed", "approved", "approve", "lgtm", "go ahead", "ship it", "yes", "perfect"]
675+
676+
const isRejectionPhrase = rejectionPhrases.some((phrase) => userText.includes(phrase))
677+
const isRejectionWord = rejectionWords.some((word) => {
678+
const regex = new RegExp(`\\b${word}\\b`)
679+
return regex.test(userText)
680+
})
681+
const isRejection = isRejectionPhrase || isRejectionWord
682+
const isApproval = !isRejection && !hasRefinementQualifier && approvalPhrases.some((phrase) => userText.includes(phrase))
683+
const action = isRejection ? "reject" : isApproval ? "approve" : "refine"
684+
Telemetry.track({
685+
type: "plan_revision",
686+
timestamp: Date.now(),
687+
session_id: sessionID,
688+
revision_number: planRevisionCount,
689+
action,
690+
})
691+
}
651692
}
652693
}
653694
// altimate_change end

0 commit comments

Comments
 (0)