Strengthen tools scenario verifications

- tool-filtering: use word-boundary grep (-w) for blacklisted tools to
  avoid false positives on substrings like 'bashing'
- no-tools: change question to directly request bash tool usage; update
  verify grep to check for inability patterns (can't, cannot, unable)
- virtual-filesystem: require both 'Virtual filesystem contents' AND
  'plan.md' in output; fix dead elif branch
- custom-agents: tighten grep to only match 'researcher' or 'Research'
  instead of also matching generic tool names
- skills: add lowercase 'skill' to grep pattern for broader matching
- mcp-servers: replace soft-pass (non-empty output) with meaningful
  content grep; add separate failure message for pattern mismatch

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: patniko

test/scenarios/tools/custom-agents/verify.sh

@@ -69,7 +69,7 @@ run_with_timeout() {
 
   # Check that the response mentions the researcher agent or its tools
   if [ "$code" -eq 0 ] && [ -n "$output" ]; then
-    if echo "$output" | grep -qi "research\|researcher\|grep\|glob\|view"; then
+    if echo "$output" | grep -qi "researcher\|Research"; then
       echo "✅ $name passed (confirmed custom agent)"
       PASS=$((PASS + 1))
     else

test/scenarios/tools/mcp-servers/verify.sh

@@ -67,13 +67,17 @@ run_with_timeout() {
 
   echo "$output"
 
-  if [ "$code" -eq 0 ] && [ -n "$output" ]; then
-    echo "✅ $name passed (got response)"
+  if [ "$code" -eq 0 ] && [ -n "$output" ] && echo "$output" | grep -qi "MCP\|mcp\|capital\|France\|Paris\|configured"; then
+    echo "✅ $name passed (got meaningful response)"
     PASS=$((PASS + 1))
   elif [ "$code" -eq 124 ]; then
     echo "❌ $name failed (timed out after ${TIMEOUT}s)"
     FAIL=$((FAIL + 1))
     ERRORS="$ERRORS\n  - $name (timeout)"
+  elif [ "$code" -eq 0 ]; then
+    echo "❌ $name failed (expected pattern not found)"
+    FAIL=$((FAIL + 1))
+    ERRORS="$ERRORS\n  - $name"
   else
     echo "❌ $name failed (exit code $code)"
     FAIL=$((FAIL + 1))

test/scenarios/tools/no-tools/csharp/Program.cs

@@ -30,7 +30,7 @@ You can only respond with text based on your training data.
 
     var response = await session.SendAndWaitAsync(new MessageOptions
     {
-        Prompt = "What tools do you have available? List them.",
+        Prompt = "Use the bash tool to run 'echo hello'.",
     });
 
     if (response != null)

test/scenarios/tools/no-tools/go/main.go

@@ -39,7 +39,7 @@ func main() {
 	defer session.Destroy()
 
 	response, err := session.SendAndWait(ctx, copilot.MessageOptions{
-		Prompt: "What tools do you have available? List them.",
+		Prompt: "Use the bash tool to run 'echo hello'.",
 	})
 	if err != nil {
 		log.Fatal(err)

test/scenarios/tools/no-tools/python/main.py

@@ -24,7 +24,7 @@ async def main():
         )
 
         response = await session.send_and_wait(
-            {"prompt": "What tools do you have available? List them."}
+            {"prompt": "Use the bash tool to run 'echo hello'."}
         )
 
         if response:

test/scenarios/tools/no-tools/typescript/src/index.ts

@@ -19,7 +19,7 @@ async function main() {
     });
 
     const response = await session.sendAndWait({
-      prompt: "What tools do you have available? List them.",
+      prompt: "Use the bash tool to run 'echo hello'.",
     });
 
     if (response) {

test/scenarios/tools/no-tools/verify.sh

@@ -69,7 +69,7 @@ run_with_timeout() {
 
   # Check that the response indicates no tools are available
   if [ "$code" -eq 0 ] && [ -n "$output" ]; then
-    if echo "$output" | grep -qi "no tool\|not have\|don't have\|do not have\|no .* tools\|cannot\|not available\|none"; then
+    if echo "$output" | grep -qi "no tool\|can't\|cannot\|unable\|don't have\|do not have\|not available"; then
       echo "✅ $name passed (confirmed no tools)"
       PASS=$((PASS + 1))
     else

test/scenarios/tools/skills/verify.sh

@@ -68,7 +68,7 @@ run_with_timeout() {
   echo "$output"
 
   if [ "$code" -eq 0 ] && [ -n "$output" ]; then
-    if echo "$output" | grep -qi "Skill directories configured\|Alice\|greeting"; then
+    if echo "$output" | grep -qi "skill\|Skill\|greeting\|Alice"; then
       echo "✅ $name passed (confirmed skill execution)"
       PASS=$((PASS + 1))
     else

test/scenarios/tools/tool-filtering/verify.sh

@@ -75,7 +75,7 @@ run_with_timeout() {
     if echo "$output" | grep -qi "grep\|glob\|view"; then
       has_whitelisted=true
     fi
-    if echo "$output" | grep -qi "bash\|edit\|create_file"; then
+    if echo "$output" | grep -qiw "bash\|edit\|create_file"; then
       has_blacklisted=true
     fi
 

test/scenarios/tools/virtual-filesystem/verify.sh

@@ -68,10 +68,10 @@ run_with_timeout() {
   echo "$output"
 
   if [ "$code" -eq 0 ] && [ -n "$output" ]; then
-    if echo "$output" | grep -qi "Virtual filesystem contents"; then
+    if echo "$output" | grep -qi "Virtual filesystem contents" && echo "$output" | grep -qi "plan\.md"; then
       echo "✅ $name passed (virtual FS operations confirmed)"
       PASS=$((PASS + 1))
-    elif [ "$code" -eq 0 ] && [ -n "$output" ]; then
+    else
       echo "❌ $name failed (expected pattern not found)"
       FAIL=$((FAIL + 1))
       ERRORS="$ERRORS\n  - $name"