fix: resolve nightly evaluation failures

- Fix ReferenceError in gemini-scheduled-triage.eval.ts by assigning stdout
- Mitigate string length limit in test-rig.ts using Buffer for logs
- Increase telemetry log safety limit to 500MB
- Fix mock MCP server dependency resolution by using absolute paths and NODE_PATH
- Update gemini-plan-execute.eval.ts to mock issue comments for plan verification

Author: cocosheng-g

evals/gemini-plan-execute.eval.ts

@@ -27,6 +27,15 @@ describe('Gemini Plan Execution Workflow', () => {
           join(rig.testDir, '.gemini/commands/gemini-plan-execute.toml'),
         );
 
+        if (item.inputs.DESCRIPTION.includes('AI Assistant: Plan of Action')) {
+          rig.createFile(
+            'mock-data.json',
+            JSON.stringify({
+              'issue_read.get_comments': [{ body: item.inputs.DESCRIPTION }],
+            }),
+          );
+        }
+
         const stdout = await rig.run(
           ['--prompt', '/gemini-plan-execute', '--yolo'],
           item.inputs,

evals/gemini-scheduled-triage.eval.ts

@@ -31,7 +31,10 @@ describe('Scheduled Triage Workflow', () => {
           GITHUB_ENV: envFile,
         };
 
-        await rig.run(['--prompt', '/gemini-scheduled-triage', '--yolo'], env);
+        const stdout = await rig.run(
+          ['--prompt', '/gemini-scheduled-triage', '--yolo'],
+          env,
+        );
 
         const content = readFileSync(envFile, 'utf-8');
         const triagedLine = content

evals/mock-mcp-server.ts

@@ -209,7 +209,36 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
 
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
   log(`Calling tool: ${request.params.name}`);
-  const pull_number = (request.params.arguments as any)?.pull_number;
+  const args = request.params.arguments as any;
+  const pull_number = args?.pull_number;
+
+  // Try to read custom mock data from CWD
+  let customMockData: Record<string, any> = {};
+  try {
+    if (fs.existsSync('mock-data.json')) {
+      customMockData = JSON.parse(fs.readFileSync('mock-data.json', 'utf-8'));
+    }
+  } catch (err) {
+    log(`Error reading custom mock data: ${err}`);
+  }
+
+  if (customMockData[request.params.name]) {
+    const customResult = customMockData[request.params.name];
+    // If it's a function, it's not supported by JSON.
+    // If it's an array, we might want to return a specific item based on args,
+    // but for now let's just return the whole thing as content.
+    return {
+      content: [
+        {
+          type: 'text',
+          text:
+            typeof customResult === 'string'
+              ? customResult
+              : JSON.stringify(customResult),
+        },
+      ],
+    };
+  }
 
   switch (request.params.name) {
     case 'search_code':

evals/test-rig.ts

@@ -6,6 +6,7 @@ import {
   existsSync,
   rmSync,
   realpathSync,
+  statSync,
 } from 'node:fs';
 import { join, dirname, basename } from 'node:path';
 import * as os from 'node:os';
@@ -80,11 +81,16 @@ export class TestRig {
   }
 
   setupMockMcp() {
+    const rootDir = realpathSync(join(__dirname, '..'));
+    const tsxPath = join(rootDir, 'node_modules', '.bin', 'tsx');
     const mockServerPath = realpathSync(join(__dirname, 'mock-mcp-server.ts'));
     this.mcpServers['github'] = {
-      command: 'npx',
-      args: ['tsx', mockServerPath],
+      command: tsxPath,
+      args: [mockServerPath],
       trust: true,
+      env: {
+        NODE_PATH: join(rootDir, 'node_modules'),
+      },
     };
     this._setupSettings(); // Re-write with MCP config
   }
@@ -178,7 +184,15 @@ export class TestRig {
 
   readToolLogs() {
     if (!existsSync(this.telemetryLog)) return [];
-    const content = readFileSync(this.telemetryLog, 'utf-8');
+    const stats = statSync(this.telemetryLog);
+    if (stats.size > 500 * 1024 * 1024) {
+      throw new Error(
+        `Telemetry log file is too large (${stats.size} bytes). Possible infinite loop.`,
+      );
+    }
+    // Use Buffer to avoid string length limits for very large logs
+    const buffer = readFileSync(this.telemetryLog);
+    const content = buffer.toString('utf-8');
     return content
       .split(/(?<=})\s*(?={)/)
       .map((obj) => {