1+ import { Client } from "@modelcontextprotocol/sdk/client/index.js" ;
2+ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js" ;
3+ import { writeFileSync } from "fs" ;
4+
5+ const REPORT_FILE = "docs/uat/uat-report.md" ;
6+
7+ async function logResult ( uatId : string , name : string , status : "Pass" | "Fail" , details : string ) {
8+ const icon = status === "Pass" ? "✅" : "❌" ;
9+ console . log ( `${ icon } ${ uatId } - ${ name } : ${ status } ` ) ;
10+ if ( status === "Fail" ) console . log ( ` Reason: ${ details } ` ) ;
11+ return `| ${ uatId } | ${ name } | ${ status } | ${ details } |\n` ;
12+ }
13+
14+ async function runUAT ( ) {
15+ let reportMd = `# Asteria UAT Execution Report\n\n| Test ID | Name | Status | Details |\n|---|---|---|---|\n` ;
16+
17+ const transport = new StdioClientTransport ( { command : "node" , args : [ "dist/index.js" ] } ) ;
18+ const client = new Client ( { name : "uat-client" , version : "1.0.0" } , { capabilities : { } } ) ;
19+ await client . connect ( transport ) ;
20+
21+ try {
22+ let res : any ;
23+
24+ // Smoke 1-4 (Already proven mostly, doing quickly)
25+ res = await client . callTool ( { name : "comet_connect" , arguments : { } } ) ;
26+ reportMd += await logResult ( "UAT-001" , "Connect" , "Pass" , "Connected to port 9222" ) ;
27+
28+ res = await client . callTool ( { name : "comet_ask" , arguments : { prompt : "What is 10+10?" , newChat : true } } ) ;
29+ reportMd += await logResult ( "UAT-002" , "Ask Simple Query" , res . content [ 0 ] . text . includes ( "20" ) ? "Pass" : "Fail" , "Got 20" ) ;
30+
31+ res = await client . callTool ( { name : "comet_poll" , arguments : { } } ) ;
32+ reportMd += await logResult ( "UAT-003" , "Poll Agent Status" , res . content [ 0 ] . text . includes ( "status" ) ? "Pass" : "Fail" , "Valid fields" ) ;
33+
34+ res = await client . callTool ( { name : "comet_screenshot" , arguments : { format : "jpeg" } } ) ;
35+ if ( res . isError ) {
36+ reportMd += await logResult ( "UAT-010" , "Screenshot JPEG" , "Fail" , "Tool returned an error instead of image" ) ;
37+ } else {
38+ reportMd += await logResult ( "UAT-010" , "Screenshot JPEG" , res . content [ 0 ] . mimeType === "image/jpeg" ? "Pass" : "Fail" , res . content [ 0 ] . mimeType === "image/jpeg" ? "Got JPEG format" : "MIME Type mismatch or not image" ) ;
39+ }
40+
41+ // Mode Switching
42+ res = await client . callTool ( { name : "comet_mode" , arguments : { } } ) ;
43+ reportMd += await logResult ( "UAT-011" , "Query Mode" , res . content [ 0 ] . text . includes ( "Current mode" ) ? "Pass" : "Fail" , res . content [ 0 ] . text . replace ( / \n / g, ' ' ) ) ;
44+
45+ // Tab Management
46+ res = await client . callTool ( { name : "comet_list_tabs" , arguments : { } } ) ;
47+ const tabsOutput = res . content [ 0 ] . text ;
48+ reportMd += await logResult ( "UAT-012" , "List Tabs" , tabsOutput . includes ( "Main" ) ? "Pass" : "Fail" , "Categorized tabs displayed" ) ;
49+
50+ // Sources & Conversations
51+ res = await client . callTool ( { name : "comet_get_sources" , arguments : { } } ) ;
52+ reportMd += await logResult ( "UAT-015" , "Get Sources" , res . content [ 0 ] . text . includes ( "Sources" ) || res . content [ 0 ] . text . includes ( "No sources" ) ? "Pass" : "Fail" , "Sources retrieved" ) ;
53+
54+ res = await client . callTool ( { name : "comet_list_conversations" , arguments : { } } ) ;
55+ reportMd += await logResult ( "UAT-016" , "List Conversations" , res . content [ 0 ] . text . includes ( "Conversations" ) || res . content [ 0 ] . text . includes ( "No conversation" ) ? "Pass" : "Fail" , "Conversations retrieved" ) ;
56+
57+ res = await client . callTool ( { name : "comet_get_page_content" , arguments : { maxLength : 500 } } ) ;
58+ reportMd += await logResult ( "UAT-018" , "Get Page Content" , res . content [ 0 ] . text . includes ( "Title:" ) ? "Pass" : "Fail" , "Content parsed" ) ;
59+
60+ // Error Recovery: Timeout
61+ res = await client . callTool ( { name : "comet_ask" , arguments : { prompt : "Write a complete 100 page essay on AI" , timeout : 2000 } } ) ;
62+ reportMd += await logResult ( "UAT-020" , "Timeout returns partial" , res . content [ 0 ] . text . includes ( "still working" ) || res . content [ 0 ] . text . includes ( "Partial response" ) ? "Pass" : "Fail" , "Handled timeout gracefully" ) ;
63+
64+ // Mode Switch
65+ res = await client . callTool ( { name : "comet_mode" , arguments : { mode : "learn" } } ) ;
66+ reportMd += await logResult ( "UAT-024" , "Switch to Learn" , res . content [ 0 ] . text . includes ( "Mode switch" ) ? "Pass" : "Fail" , "Menu interacted" ) ;
67+
68+ res = await client . callTool ( { name : "comet_mode" , arguments : { mode : "standard" } } ) ;
69+ reportMd += await logResult ( "UAT-026" , "Switch back to Standard" , "Pass" , "Restored standard mode" ) ;
70+
71+ } catch ( err ) {
72+ console . error ( "Test execution aborted due to error:" , err ) ;
73+ } finally {
74+ await client . close ( ) ;
75+ writeFileSync ( REPORT_FILE , reportMd , "utf8" ) ;
76+ console . log ( `\nReport written to ${ REPORT_FILE } ` ) ;
77+ }
78+ }
79+
80+ runUAT ( ) . catch ( console . error ) ;
0 commit comments