Skip to content

Commit eb004c0

Browse files
author
alpsla
committed
Session 92: Add tool validation test for spectral and graphql-cop
1 parent 701eab6 commit eb004c0

3 files changed

Lines changed: 410 additions & 60 deletions

File tree

‎packages/agents/tests/integration/test-tool-validation.ts‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
/**
2+
* Tool Validation Test - Session 92
3+
*
4+
* Tests all tools to ensure they produce at least 1 finding.
5+
* Uses specific repos chosen to trigger each tool.
6+
*/
7+
8+
import * as path from 'path';
9+
import * as fs from 'fs';
10+
import { execSync } from 'child_process';
11+
import { JavaToolOrchestrator } from '../../src/two-branch/tools/java/java-tool-orchestrator';
12+
13+
// Test scenarios for tool validation
14+
interface ToolValidationScenario {
15+
name: string;
16+
repoUrl: string;
17+
prNumber?: number;
18+
language: 'java' | 'typescript' | 'python';
19+
expectedTools: string[]; // Tools expected to find issues
20+
reason: string; // Why this repo triggers these tools
21+
}
22+
23+
const TOOL_VALIDATION_SCENARIOS: ToolValidationScenario[] = [
24+
// Swagger Petstore - Has openapi.yaml for Spectral
25+
{
26+
name: 'Swagger Petstore (OpenAPI for Spectral)',
27+
repoUrl: 'https://github.com/swagger-api/swagger-petstore',
28+
prNumber: 218,
29+
language: 'java',
30+
expectedTools: ['spectral'],
31+
reason: 'Has src/main/resources/openapi.yaml'
32+
},
33+
// Netflix DGS Examples - Has GraphQL schemas for graphql-cop
34+
{
35+
name: 'Netflix DGS Examples (GraphQL for graphql-cop)',
36+
repoUrl: 'https://github.com/Netflix/dgs-examples-java',
37+
prNumber: 196,
38+
language: 'java',
39+
expectedTools: ['graphql-cop'],
40+
reason: 'Has .graphqls schema files'
41+
}
42+
];
43+
44+
/**
45+
* Clone a repository
46+
*/
47+
function cloneRepository(repoUrl: string, targetPath: string): void {
48+
console.log(` 🔄 Cloning ${repoUrl}...`);
49+
50+
if (fs.existsSync(targetPath)) {
51+
execSync(`rm -rf ${targetPath}`);
52+
}
53+
54+
execSync(`git clone --depth 1 ${repoUrl} ${targetPath}`, {
55+
stdio: 'pipe',
56+
encoding: 'utf-8'
57+
});
58+
59+
console.log(` ✅ Repository cloned to ${targetPath}`);
60+
}
61+
62+
/**
63+
* Check out a specific PR
64+
*/
65+
function checkoutPR(repoPath: string, prNumber: number): void {
66+
console.log(` 🔀 Checking out PR #${prNumber}...`);
67+
68+
try {
69+
execSync(`git -C ${repoPath} fetch origin pull/${prNumber}/head:pr-${prNumber}`, { stdio: 'pipe' });
70+
execSync(`git -C ${repoPath} checkout pr-${prNumber}`, { stdio: 'pipe' });
71+
console.log(` ✅ Checked out PR branch`);
72+
} catch (error) {
73+
console.log(` ⚠️ Could not checkout PR, using default branch`);
74+
}
75+
}
76+
77+
/**
78+
* Run tool validation for a scenario
79+
*/
80+
async function runToolValidation(scenario: ToolValidationScenario): Promise<{
81+
success: boolean;
82+
toolResults: Record<string, number>;
83+
}> {
84+
const timestamp = Date.now();
85+
const repoPath = `/tmp/tool-validation-${timestamp}`;
86+
87+
console.log(`\n${'='.repeat(80)}`);
88+
console.log(`🧪 Testing: ${scenario.name}`);
89+
console.log(`📋 Reason: ${scenario.reason}`);
90+
console.log(`🎯 Expected tools: ${scenario.expectedTools.join(', ')}`);
91+
console.log(`${'='.repeat(80)}\n`);
92+
93+
try {
94+
// Clone repository
95+
cloneRepository(scenario.repoUrl, repoPath);
96+
97+
// Checkout PR if specified
98+
if (scenario.prNumber) {
99+
checkoutPR(repoPath, scenario.prNumber);
100+
}
101+
102+
// List files to verify content
103+
console.log(`\n📁 Checking for relevant files...`);
104+
105+
// Check for OpenAPI files
106+
const openApiFiles = execSync(
107+
`find ${repoPath} -name "*.yaml" -o -name "*.yml" -o -name "*.json" 2>/dev/null | xargs grep -l "openapi\\|swagger" 2>/dev/null | head -5 || true`,
108+
{ encoding: 'utf-8' }
109+
).trim();
110+
if (openApiFiles) {
111+
console.log(` 📋 OpenAPI files found:\n${openApiFiles.split('\n').map(f => ` - ${f}`).join('\n')}`);
112+
}
113+
114+
// Check for GraphQL files
115+
const graphqlFiles = execSync(
116+
`find ${repoPath} -name "*.graphql" -o -name "*.graphqls" 2>/dev/null | head -5 || true`,
117+
{ encoding: 'utf-8' }
118+
).trim();
119+
if (graphqlFiles) {
120+
console.log(` 🔮 GraphQL files found:\n${graphqlFiles.split('\n').map(f => ` - ${f}`).join('\n')}`);
121+
}
122+
123+
// Check for pom.xml (dependency-check)
124+
const pomFiles = execSync(
125+
`find ${repoPath} -name "pom.xml" 2>/dev/null | head -5 || true`,
126+
{ encoding: 'utf-8' }
127+
).trim();
128+
if (pomFiles) {
129+
console.log(` 📦 Maven POM files found:\n${pomFiles.split('\n').map(f => ` - ${f}`).join('\n')}`);
130+
}
131+
132+
// Run the orchestrator
133+
console.log(`\n🚀 Running Java Tool Orchestrator...`);
134+
const orchestrator = new JavaToolOrchestrator();
135+
136+
const result = await orchestrator.orchestrate({
137+
repositoryPath: repoPath,
138+
branch: scenario.prNumber ? `pr-${scenario.prNumber}` : 'main',
139+
mode: 'complete', // Use complete mode to run all tools
140+
tier: 'basic'
141+
});
142+
143+
// Collect tool results
144+
const toolResults: Record<string, number> = {};
145+
146+
for (const toolResult of result.results) {
147+
toolResults[toolResult.tool] = toolResult.issues.length;
148+
}
149+
150+
console.log(`\n📊 Tool Results:`);
151+
console.log(`${'─'.repeat(50)}`);
152+
for (const [tool, count] of Object.entries(toolResults).sort((a, b) => b[1] - a[1])) {
153+
const status = count > 0 ? '✅' : '❌';
154+
const isExpected = scenario.expectedTools.includes(tool);
155+
const highlight = isExpected ? (count > 0 ? ' ⭐ EXPECTED!' : ' ⚠️ MISSING!') : '';
156+
console.log(` ${status} ${tool}: ${count} issues${highlight}`);
157+
}
158+
159+
// Check if expected tools found issues
160+
const expectedSuccess = scenario.expectedTools.every(tool => (toolResults[tool] || 0) > 0);
161+
162+
console.log(`\n${expectedSuccess ? '✅' : '❌'} Expected tools ${expectedSuccess ? 'ALL FOUND ISSUES' : 'SOME MISSING'}`);
163+
164+
// Cleanup
165+
execSync(`rm -rf ${repoPath}`);
166+
167+
return {
168+
success: expectedSuccess,
169+
toolResults
170+
};
171+
172+
} catch (error) {
173+
console.error(`\n❌ Error: ${error}`);
174+
execSync(`rm -rf ${repoPath} 2>/dev/null || true`);
175+
return {
176+
success: false,
177+
toolResults: {}
178+
};
179+
}
180+
}
181+
182+
/**
183+
* Main test runner
184+
*/
185+
async function main(): Promise<void> {
186+
console.log(`
187+
╔═══════════════════════════════════════════════════════════════════════════╗
188+
║ TOOL VALIDATION TEST - SESSION 92 ║
189+
╠═══════════════════════════════════════════════════════════════════════════╣
190+
║ Purpose: Ensure all tools can produce at least 1 finding ║
191+
║ Tools being validated: SpotBugs, JDepend, dependency-check, ║
192+
║ spectral, graphql-cop ║
193+
╚═══════════════════════════════════════════════════════════════════════════╝
194+
`);
195+
196+
const results: Array<{
197+
scenario: string;
198+
success: boolean;
199+
toolResults: Record<string, number>;
200+
}> = [];
201+
202+
for (const scenario of TOOL_VALIDATION_SCENARIOS) {
203+
const result = await runToolValidation(scenario);
204+
results.push({
205+
scenario: scenario.name,
206+
...result
207+
});
208+
}
209+
210+
// Final summary
211+
console.log(`\n${'═'.repeat(80)}`);
212+
console.log(` FINAL VALIDATION SUMMARY`);
213+
console.log(`${'═'.repeat(80)}\n`);
214+
215+
// Aggregate all tool findings
216+
const allToolResults: Record<string, number> = {};
217+
for (const result of results) {
218+
for (const [tool, count] of Object.entries(result.toolResults)) {
219+
allToolResults[tool] = (allToolResults[tool] || 0) + count;
220+
}
221+
}
222+
223+
console.log(`📊 Aggregate Tool Findings:`);
224+
console.log(`${'─'.repeat(50)}`);
225+
226+
const toolsToValidate = ['spotbugs', 'jdepend', 'dependency-check', 'spectral', 'graphql-cop'];
227+
let allValidated = true;
228+
229+
for (const tool of toolsToValidate) {
230+
const count = allToolResults[tool] || 0;
231+
const status = count > 0 ? '✅' : '❌';
232+
console.log(` ${status} ${tool}: ${count} total issues`);
233+
if (count === 0) allValidated = false;
234+
}
235+
236+
console.log(`\n${'─'.repeat(50)}`);
237+
console.log(`${allValidated ? '✅ ALL TOOLS VALIDATED' : '⚠️ SOME TOOLS NEED ATTENTION'}`);
238+
console.log(`${'═'.repeat(80)}\n`);
239+
240+
process.exit(allValidated ? 0 : 1);
241+
}
242+
243+
main().catch(console.error);

‎rex-session-92-tool-validation.md‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Session 92: Tool Validation - Ensure All Tools Produce Findings
2+
3+
## Goal
4+
Verify that all 13 tools are configured correctly by finding test PRs that produce at least 1 finding per tool.
5+
6+
## Tools Currently Showing 0 Findings
7+
1. **SpotBugs** - Bug Detection (needs compiled Java with bugs)
8+
2. **JDepend** - Architecture (needs compiled Java classes)
9+
3. **dependency-check** - Dependency Scan (needs vulnerable dependencies)
10+
4. **spectral** - API Schema (needs OpenAPI/Swagger files)
11+
5. **graphql-cop** - GraphQL Security (needs GraphQL schema/endpoints)
12+
13+
---
14+
15+
### 1. Find and Test SpotBugs Detection
16+
**Goal**: Find a Java PR with actual bugs that SpotBugs can detect
17+
**Steps**:
18+
1. Search for Java repos with known SpotBugs issues or buggy code
19+
2. Look for repos like `spotbugs/spotbugs-samples` or Java projects with null pointer issues
20+
3. Run test with `test-v9-lite-e2e.ts` to verify SpotBugs finds issues
21+
4. Document the test PR that produces SpotBugs findings
22+
23+
### 2. Find and Test JDepend Detection
24+
**Goal**: Find a Java PR with architecture/dependency issues
25+
**Steps**:
26+
1. JDepend analyzes package dependencies and cycles
27+
2. Find a Java project with circular dependencies or poor package structure
28+
3. Verify JDepend produces architecture metrics
29+
4. Document findings
30+
31+
### 3. Find and Test dependency-check Detection
32+
**Goal**: Find a PR with known CVE vulnerabilities in dependencies
33+
**Steps**:
34+
1. Search for Java projects with outdated/vulnerable dependencies
35+
2. Look for repos with known Log4j, Spring, or Jackson vulnerabilities
36+
3. Test with dependency-check and verify CVE detection
37+
4. Document the vulnerable dependencies found
38+
39+
### 4. Find and Test Spectral API Schema Detection
40+
**Goal**: Find a PR with OpenAPI/Swagger specs that have issues
41+
**Steps**:
42+
1. Search for repos with `openapi.yaml`, `swagger.json`, or `api-spec` files
43+
2. Look for API-first projects like `swagger-api/swagger-petstore`
44+
3. Run test and verify Spectral finds API schema issues
45+
4. Document the API validation findings
46+
47+
### 5. Find and Test graphql-cop Detection
48+
**Goal**: Find a PR with GraphQL schema/endpoint security issues
49+
**Steps**:
50+
1. Search for repos with GraphQL schemas (`.graphql` files, `schema.graphql`)
51+
2. Look for GraphQL servers like `graphql-java/graphql-java` or Apollo projects
52+
3. Run test and verify graphql-cop finds security issues
53+
4. Document the GraphQL security findings
54+
55+
### 6. Create Comprehensive Tool Validation Report
56+
**Goal**: Document all tool validation results
57+
**Steps**:
58+
1. Create a markdown report showing each tool with example findings
59+
2. List test repositories used for each tool
60+
3. Confirm all 13 tools are properly configured
61+
4. Update CLAUDE.md if any tool configuration issues found
62+
63+
---
64+
65+
## Test Repositories to Consider
66+
67+
| Tool | Potential Test Repo | Why |
68+
|------|---------------------|-----|
69+
| SpotBugs | `iluwatar/java-design-patterns` | Large Java codebase |
70+
| JDepend | `spring-projects/spring-framework` | Complex package structure |
71+
| dependency-check | Older Java projects with Log4j 2.x | Known CVEs |
72+
| spectral | `swagger-api/swagger-petstore` | OpenAPI specs |
73+
| graphql-cop | `graphql-java/graphql-java` | GraphQL schemas |
74+
75+
## Success Criteria
76+
- [ ] SpotBugs: >= 1 bug finding
77+
- [ ] JDepend: >= 1 architecture metric
78+
- [ ] dependency-check: >= 1 CVE finding
79+
- [ ] spectral: >= 1 API schema issue
80+
- [ ] graphql-cop: >= 1 GraphQL security issue

0 commit comments

Comments
 (0)