|
| 1 | +#!/usr/bin/env node |
| 2 | +/** |
| 3 | + * javacg-static vs Codegraph: Java fixture call graph comparison |
| 4 | + * |
| 5 | + * Runs javacg-static (gousiosg/java-callgraph) on the compiled fixture JAR, |
| 6 | + * parses its output, maps class:method names to ClassName.method form, and |
| 7 | + * computes precision/recall against expected-edges.json. |
| 8 | + * |
| 9 | + * javacg-static output format: |
| 10 | + * M:pkg.ClassName:method(argDescriptors) (T)pkg.ClassName:method(argDescriptors) |
| 11 | + * where T is: C=virtual, S=static, O=special (constructors, super), I=interface, D=dynamic |
| 12 | + * |
| 13 | + * Name mapping to expected-edges.json convention: |
| 14 | + * source <init> → ClassName.ClassName (constructor-as-method) |
| 15 | + * target <init> → ClassName (constructor target = class name only) |
| 16 | + * other method → ClassName.method |
| 17 | + * |
| 18 | + * Prerequisites: |
| 19 | + * 1. Java runtime (java -jar must work) |
| 20 | + * 2. javacg-static JAR — download from: |
| 21 | + * https://github.com/gousiosg/java-callgraph/releases |
| 22 | + * or build: `cd java-callgraph && mvn package -DskipTests` |
| 23 | + * Pass via --jar or set JAVACG_JAR, or place at scripts/lib/javacg-static.jar |
| 24 | + * 3. Compiled fixture JAR: |
| 25 | + * cd tests/benchmarks/resolution/fixtures/java && make |
| 26 | + * |
| 27 | + * Usage: |
| 28 | + * node scripts/compare-javacg.mjs |
| 29 | + * node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar |
| 30 | + * node scripts/compare-javacg.mjs --json |
| 31 | + */ |
| 32 | + |
| 33 | +import { execFileSync } from 'node:child_process'; |
| 34 | +import fs from 'node:fs'; |
| 35 | +import path from 'node:path'; |
| 36 | +import { fileURLToPath } from 'node:url'; |
| 37 | + |
| 38 | +const __dirname = path.dirname(fileURLToPath(import.meta.url)); |
| 39 | +const ROOT = path.resolve(__dirname, '..'); |
| 40 | +const FIXTURE_DIR = path.join(ROOT, 'tests/benchmarks/resolution/fixtures/java'); |
| 41 | + |
| 42 | +// ── CLI ────────────────────────────────────────────────────────────────────── |
| 43 | + |
| 44 | +const args = process.argv.slice(2); |
| 45 | +const jsonFlag = args.includes('--json'); |
| 46 | +const jarArgIdx = args.indexOf('--jar'); |
| 47 | +const jarArgPath = jarArgIdx !== -1 ? args[jarArgIdx + 1] : null; |
| 48 | + |
| 49 | +// ── Tool discovery ─────────────────────────────────────────────────────────── |
| 50 | + |
| 51 | +function findJavacgJar() { |
| 52 | + if (jarArgPath) return jarArgPath; |
| 53 | + if (process.env.JAVACG_JAR) return process.env.JAVACG_JAR; |
| 54 | + // Glob for any jar with "javacg" in the name under scripts/lib/ |
| 55 | + const libDir = path.join(__dirname, 'lib'); |
| 56 | + if (fs.existsSync(libDir)) { |
| 57 | + const jar = fs.readdirSync(libDir).find((f) => f.includes('javacg') && f.endsWith('.jar')); |
| 58 | + if (jar) return path.join(libDir, jar); |
| 59 | + } |
| 60 | + return null; |
| 61 | +} |
| 62 | + |
| 63 | +// ── Name mapping ───────────────────────────────────────────────────────────── |
| 64 | + |
| 65 | +/** |
| 66 | + * Scan .java source files to build SimpleClassName → filename.java map. |
| 67 | + * Used to resolve file fields in the edge key format "name@file". |
| 68 | + */ |
| 69 | +function buildClassFileMap(fixtureDir) { |
| 70 | + const map = new Map(); |
| 71 | + for (const filename of fs.readdirSync(fixtureDir)) { |
| 72 | + if (!filename.endsWith('.java')) continue; |
| 73 | + const src = fs.readFileSync(path.join(fixtureDir, filename), 'utf8'); |
| 74 | + const m = src.match(/(?:public\s+)?(?:abstract\s+)?(?:class|interface)\s+(\w+)/); |
| 75 | + if (m) map.set(m[1], filename); |
| 76 | + } |
| 77 | + return map; |
| 78 | +} |
| 79 | + |
| 80 | +/** |
| 81 | + * Parse "pkg.ClassName:methodName(descriptors)" into { className, methodName }. |
| 82 | + * Works with both "." and "/" as package separators (javacg uses "."). |
| 83 | + */ |
| 84 | +function parseMethodSpec(spec) { |
| 85 | + // Strip argument descriptor — everything from "(" onwards |
| 86 | + const parenIdx = spec.indexOf('('); |
| 87 | + const withoutArgs = parenIdx !== -1 ? spec.slice(0, parenIdx) : spec; |
| 88 | + const colonIdx = withoutArgs.indexOf(':'); |
| 89 | + if (colonIdx === -1) return null; |
| 90 | + const classPart = withoutArgs.slice(0, colonIdx); |
| 91 | + const methodName = withoutArgs.slice(colonIdx + 1); |
| 92 | + // Simple class name: last segment after "." or "/" |
| 93 | + const className = classPart.split(/[./]/).at(-1); |
| 94 | + if (!className) return null; |
| 95 | + return { className, methodName }; |
| 96 | +} |
| 97 | + |
| 98 | +/** Source side: "<init>" method maps to ClassName.ClassName. */ |
| 99 | +function toSourceName({ className, methodName }) { |
| 100 | + return methodName === '<init>' ? `${className}.${className}` : `${className}.${methodName}`; |
| 101 | +} |
| 102 | + |
| 103 | +/** Target side: "<init>" method maps to just ClassName (constructor target). */ |
| 104 | +function toTargetName({ className, methodName }) { |
| 105 | + return methodName === '<init>' ? className : `${className}.${methodName}`; |
| 106 | +} |
| 107 | + |
| 108 | +// ── Ground truth ───────────────────────────────────────────────────────────── |
| 109 | + |
| 110 | +function loadGroundTruth(fixtureDir) { |
| 111 | + const manifest = JSON.parse( |
| 112 | + fs.readFileSync(path.join(fixtureDir, 'expected-edges.json'), 'utf8'), |
| 113 | + ); |
| 114 | + const set = new Set( |
| 115 | + manifest.edges.map( |
| 116 | + (e) => |
| 117 | + `${e.source.name}@${path.basename(e.source.file)}→${e.target.name}@${path.basename(e.target.file)}`, |
| 118 | + ), |
| 119 | + ); |
| 120 | + return set; |
| 121 | +} |
| 122 | + |
| 123 | +// ── Run javacg-static ──────────────────────────────────────────────────────── |
| 124 | + |
| 125 | +function runJavacg(javacgJar, fixtureDir) { |
| 126 | + const fixtureJar = path.join(fixtureDir, 'fixture.jar'); |
| 127 | + if (!fs.existsSync(fixtureJar)) { |
| 128 | + console.error(`fixture.jar not found at ${fixtureJar}`); |
| 129 | + console.error(`Build it with: cd ${fixtureDir} && make`); |
| 130 | + process.exit(1); |
| 131 | + } |
| 132 | + try { |
| 133 | + return execFileSync('java', ['-jar', javacgJar, fixtureJar], { |
| 134 | + encoding: 'utf8', |
| 135 | + stdio: ['ignore', 'pipe', 'pipe'], |
| 136 | + }); |
| 137 | + } catch (err) { |
| 138 | + // javacg-static may exit non-zero but still produce useful stdout |
| 139 | + if (err.stdout?.trim()) return err.stdout; |
| 140 | + console.error(`javacg-static failed: ${err.message}`); |
| 141 | + process.exit(1); |
| 142 | + } |
| 143 | +} |
| 144 | + |
| 145 | +/** |
| 146 | + * Parse javacg-static text output into a Set of edge keys. |
| 147 | + * |
| 148 | + * Line format: |
| 149 | + * M:pkg.Class:method(args) (T)pkg.Class:method(args) |
| 150 | + * |
| 151 | + * Only edges where both class names appear in classFileMap are included — |
| 152 | + * this filters out JDK / stdlib calls (HashMap, String, System.out, etc.). |
| 153 | + */ |
| 154 | +function parseJavacgOutput(output, classFileMap) { |
| 155 | + // M: caller (T) callee — the space between caller and (T) may vary |
| 156 | + const lineRe = /^M:(\S+)\s+\(([CSOIDE])\)(\S+)$/; |
| 157 | + const edges = new Set(); |
| 158 | + |
| 159 | + for (const rawLine of output.split('\n')) { |
| 160 | + const line = rawLine.trim(); |
| 161 | + if (!line.startsWith('M:')) continue; |
| 162 | + |
| 163 | + const m = line.match(lineRe); |
| 164 | + if (!m) continue; |
| 165 | + |
| 166 | + const [, sourceSpec, , targetSpec] = m; |
| 167 | + |
| 168 | + const sourceParsed = parseMethodSpec(sourceSpec); |
| 169 | + const targetParsed = parseMethodSpec(targetSpec); |
| 170 | + if (!sourceParsed || !targetParsed) continue; |
| 171 | + |
| 172 | + const sourceFile = classFileMap.get(sourceParsed.className); |
| 173 | + const targetFile = classFileMap.get(targetParsed.className); |
| 174 | + // Skip edges to/from classes outside the fixture (JDK, etc.) |
| 175 | + if (!sourceFile || !targetFile) continue; |
| 176 | + |
| 177 | + const sourceName = toSourceName(sourceParsed); |
| 178 | + const targetName = toTargetName(targetParsed); |
| 179 | + |
| 180 | + const key = `${sourceName}@${sourceFile}→${targetName}@${targetFile}`; |
| 181 | + // Skip self-edges (e.g. recursive calls not in expected-edges) |
| 182 | + if (sourceName === targetName && sourceFile === targetFile) continue; |
| 183 | + edges.add(key); |
| 184 | + } |
| 185 | + return edges; |
| 186 | +} |
| 187 | + |
| 188 | +// ── Metrics ────────────────────────────────────────────────────────────────── |
| 189 | + |
| 190 | +function computeMetrics(predicted, groundTruth) { |
| 191 | + let tp = 0; |
| 192 | + const fp = []; |
| 193 | + const fn = []; |
| 194 | + for (const edge of predicted) (groundTruth.has(edge) ? tp++ : fp.push(edge)); |
| 195 | + for (const edge of groundTruth) if (!predicted.has(edge)) fn.push(edge); |
| 196 | + return { |
| 197 | + precision: predicted.size === 0 ? 0 : tp / predicted.size, |
| 198 | + recall: groundTruth.size === 0 ? 0 : tp / groundTruth.size, |
| 199 | + tp, |
| 200 | + fp: fp.length, |
| 201 | + fn: fn.length, |
| 202 | + totalPredicted: predicted.size, |
| 203 | + totalExpected: groundTruth.size, |
| 204 | + fpEdges: fp, |
| 205 | + fnEdges: fn, |
| 206 | + }; |
| 207 | +} |
| 208 | + |
| 209 | +// ── Main ───────────────────────────────────────────────────────────────────── |
| 210 | + |
| 211 | +const javacgJar = findJavacgJar(); |
| 212 | +if (!javacgJar) { |
| 213 | + console.error('javacg-static JAR not found.'); |
| 214 | + console.error('Download from: https://github.com/gousiosg/java-callgraph/releases'); |
| 215 | + console.error('Then use one of:'); |
| 216 | + console.error(' node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar'); |
| 217 | + console.error(' JAVACG_JAR=/path/to/javacg-0.1-SNAPSHOT.jar node scripts/compare-javacg.mjs'); |
| 218 | + console.error(' cp /path/to/javacg-0.1-SNAPSHOT.jar scripts/lib/javacg-static.jar'); |
| 219 | + process.exit(1); |
| 220 | +} |
| 221 | + |
| 222 | +const classFileMap = buildClassFileMap(FIXTURE_DIR); |
| 223 | +const groundTruth = loadGroundTruth(FIXTURE_DIR); |
| 224 | + |
| 225 | +console.error(`\n── JAVA ──────────────────────────────────────────────────`); |
| 226 | +console.error(` Ground truth: ${groundTruth.size} edges`); |
| 227 | +console.error(` Running javacg-static on fixture.jar...`); |
| 228 | + |
| 229 | +const rawOutput = runJavacg(javacgJar, FIXTURE_DIR); |
| 230 | +const predictedEdges = parseJavacgOutput(rawOutput, classFileMap); |
| 231 | + |
| 232 | +console.error(` javacg-static: ${predictedEdges.size} named benchmark edges`); |
| 233 | + |
| 234 | +const metrics = computeMetrics(predictedEdges, groundTruth); |
| 235 | + |
| 236 | +console.error( |
| 237 | + ` precision=${metrics.precision.toFixed(2)} recall=${metrics.recall.toFixed(2)} ` + |
| 238 | + `TP=${metrics.tp} FP=${metrics.fp} FN=${metrics.fn}`, |
| 239 | +); |
| 240 | + |
| 241 | +if (metrics.fpEdges.length) { |
| 242 | + console.error(` FP (edges not in expected-edges.json):`); |
| 243 | + for (const e of metrics.fpEdges) console.error(` - ${e}`); |
| 244 | +} |
| 245 | +if (metrics.fnEdges.length) { |
| 246 | + console.error(` FN (expected edges missed):`); |
| 247 | + for (const e of metrics.fnEdges) console.error(` - ${e}`); |
| 248 | +} |
| 249 | + |
| 250 | +if (jsonFlag) { |
| 251 | + console.log( |
| 252 | + JSON.stringify( |
| 253 | + { |
| 254 | + java: { |
| 255 | + groundTruth: groundTruth.size, |
| 256 | + javacgEdges: predictedEdges.size, |
| 257 | + metrics, |
| 258 | + }, |
| 259 | + }, |
| 260 | + null, |
| 261 | + 2, |
| 262 | + ), |
| 263 | + ); |
| 264 | +} else { |
| 265 | + console.log('\n## javacg-static vs expected-edges.json Ground Truth\n'); |
| 266 | + console.log('| Language | Tool | Precision | Recall | TP | FP | FN |'); |
| 267 | + console.log('|----------|------|:---------:|:------:|---:|---:|---:|'); |
| 268 | + console.log( |
| 269 | + `| Java | javacg-static (CHA) | ${(metrics.precision * 100).toFixed(0)}% | ` + |
| 270 | + `${(metrics.recall * 100).toFixed(0)}% | ${metrics.tp} | ${metrics.fp} | ${metrics.fn} |`, |
| 271 | + ); |
| 272 | +} |
0 commit comments