test-validation-framework.js

#!/usr/bin/env node
// Copyright 2025
// Damien Davison & Michael Maillet & Sacha Davison
// Recursive AI Devs
//
// META-TEST: Validates the validation framework itself
// Ensures test infrastructure is working correctly

console.log('='.repeat(80));
console.log('AG-TUNE VALIDATION FRAMEWORK INTEGRITY CHECK');
console.log('Meta-test: Validating the validators');
console.log('='.repeat(80));

import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

let checks = 0;
let passed = 0;

function check(name, testFn) {
  checks++;
  try {
    const result = testFn();
    if (result) {
      console.log(`✓ ${name}`);
      passed++;
    } else {
      console.log(`✗ ${name}`);
    }
    return result;
  } catch (error) {
    console.log(`✗ ${name} - Error: ${error.message}`);
    return false;
  }
}

console.log('\n[1] Test Files Exist\n');

check('Module validation test exists', () => 
  fs.existsSync(path.join(__dirname, 'test-module-validation.js'))
);

check('System invariants test exists', () => 
  fs.existsSync(path.join(__dirname, 'test-system-invariants.js'))
);

check('Ablation study test exists', () => 
  fs.existsSync(path.join(__dirname, 'test-ablation-study.js'))
);

check('Interpretability test exists', () => 
  fs.existsSync(path.join(__dirname, 'test-interpretability.js'))
);

console.log('\n[2] Documentation Exists\n');

check('VALIDATION.md exists', () => 
  fs.existsSync(path.join(__dirname, 'VALIDATION.md'))
);

check('VALIDATION_GUIDE.md exists', () => 
  fs.existsSync(path.join(__dirname, 'VALIDATION_GUIDE.md'))
);

console.log('\n[3] Package.json Integration\n');

const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'), 'utf8'));

check('test-modules script defined', () => 
  packageJson.scripts && packageJson.scripts['test-modules']
);

check('test-invariants script defined', () => 
  packageJson.scripts && packageJson.scripts['test-invariants']
);

check('test-ablation script defined', () => 
  packageJson.scripts && packageJson.scripts['test-ablation']
);

check('test-interpretability script defined', () => 
  packageJson.scripts && packageJson.scripts['test-interpretability']
);

check('test-all script defined', () => 
  packageJson.scripts && packageJson.scripts['test-all']
);

console.log('\n[4] Test File Structure\n');

function checkTestStructure(filepath, expectedStrings) {
  const content = fs.readFileSync(filepath, 'utf8');
  return expectedStrings.every(str => content.includes(str));
}

check('Module tests have KernelPCA tests', () => 
  checkTestStructure(
    path.join(__dirname, 'test-module-validation.js'),
    ['Kernel PCA', 'Reconstruction sanity', 'Trajectory smoothness']
  )
);

check('Module tests have FFT tests', () => 
  checkTestStructure(
    path.join(__dirname, 'test-module-validation.js'),
    ['FFT', 'Known metrical patterns', 'Pattern discrimination']
  )
);

check('Module tests have CYK tests', () => 
  checkTestStructure(
    path.join(__dirname, 'test-module-validation.js'),
    ['CYK', 'Fuzz testing', 'Mutation testing']
  )
);

check('Module tests have TD(λ) tests', () => 
  checkTestStructure(
    path.join(__dirname, 'test-module-validation.js'),
    ['TD', 'Learning curve', 'Reward ablation']
  )
);

check('Module tests have Cycle Detector tests', () => 
  checkTestStructure(
    path.join(__dirname, 'test-module-validation.js'),
    ['Floyd', 'Determinism', 'false positives']
  )
);

console.log('\n[5] System Invariants Coverage\n');

check('System tests check grammar', () => 
  checkTestStructure(
    path.join(__dirname, 'test-system-invariants.js'),
    ['Grammar', 'Validation']
  )
);

check('System tests check cycles', () => 
  checkTestStructure(
    path.join(__dirname, 'test-system-invariants.js'),
    ['Cycle', 'Detection']
  )
);

check('System tests check emotional continuity', () => 
  checkTestStructure(
    path.join(__dirname, 'test-system-invariants.js'),
    ['Emotional', 'Continuity']
  )
);

check('System tests check meter consistency', () => 
  checkTestStructure(
    path.join(__dirname, 'test-system-invariants.js'),
    ['Meter', 'Analysis']
  )
);

check('System tests check novelty', () => 
  checkTestStructure(
    path.join(__dirname, 'test-system-invariants.js'),
    ['Vocabulary', 'Diversity']
  )
);

console.log('\n[6] Ablation Study Coverage\n');

check('Ablation tests FFT causality', () => 
  checkTestStructure(
    path.join(__dirname, 'test-ablation-study.js'),
    ['FFT', 'Rhythm', 'collapse']
  )
);

check('Ablation tests Rete causality', () => 
  checkTestStructure(
    path.join(__dirname, 'test-ablation-study.js'),
    ['Rete', 'Constraint', 'theme']
  )
);

check('Ablation tests TD causality', () => 
  checkTestStructure(
    path.join(__dirname, 'test-ablation-study.js'),
    ['TD', 'Value', 'aesthetic']
  )
);

check('Ablation tests all 7 components', () => {
  const content = fs.readFileSync(path.join(__dirname, 'test-ablation-study.js'), 'utf8');
  const components = ['FFT', 'Rete', 'TD', 'Floyd', 'Kernel PCA', 'CYK', 'Beam'];
  return components.every(comp => content.includes(comp));
});

console.log('\n[7] Interpretability Features\n');

check('Interpretability defines reasoning trace', () => 
  checkTestStructure(
    path.join(__dirname, 'test-interpretability.js'),
    ['reasoningTrace', 'emotionalVector', 'beamCandidates']
  )
);

check('Interpretability has "Why this line?" inspector', () => 
  checkTestStructure(
    path.join(__dirname, 'test-interpretability.js'),
    ['Why this line', 'explainLineSelection']
  )
);

check('Interpretability has reward attribution', () => 
  checkTestStructure(
    path.join(__dirname, 'test-interpretability.js'),
    ['rewardAttribution', 'components', 'contribution']
  )
);

check('Interpretability has visualization data', () => 
  checkTestStructure(
    path.join(__dirname, 'test-interpretability.js'),
    ['visualization', 'timeline']
  )
);

check('Interpretability exports JSON', () => 
  checkTestStructure(
    path.join(__dirname, 'test-interpretability.js'),
    ['JSON.stringify', 'exportPath', 'reasoning-trace']
  )
);

console.log('\n[8] Documentation Quality\n');

check('VALIDATION.md has test suite descriptions', () => 
  checkTestStructure(
    path.join(__dirname, 'VALIDATION.md'),
    ['Module-Level', 'System-Level', 'Ablation', 'Interpretability']
  )
);

check('VALIDATION.md has running instructions', () => 
  checkTestStructure(
    path.join(__dirname, 'VALIDATION.md'),
    ['npm run', 'test-modules', 'test-invariants']
  )
);

check('VALIDATION.md has invariant table', () => 
  checkTestStructure(
    path.join(__dirname, 'VALIDATION.md'),
    ['Invariant', 'Test Method', 'Status']
  )
);

check('VALIDATION.md has component causality matrix', () => 
  checkTestStructure(
    path.join(__dirname, 'VALIDATION.md'),
    ['Component', 'Causal', 'Degradation', 'Severity']
  )
);

check('VALIDATION_GUIDE.md has debugging section', () => 
  checkTestStructure(
    path.join(__dirname, 'VALIDATION_GUIDE.md'),
    ['Debugging Test Failures', 'Step 1', 'Identify']
  )
);

check('VALIDATION_GUIDE.md has FAQ', () => 
  checkTestStructure(
    path.join(__dirname, 'VALIDATION_GUIDE.md'),
    ['FAQ', 'Q:', 'A:']
  )
);

console.log('\n[9] Test File Executability\n');

check('Module test is executable', () => {
  // Files are executable via node (shebang), permission bits may vary by OS
  return fs.existsSync(path.join(__dirname, 'test-module-validation.js'));
});

check('Ablation test is executable', () => {
  // Files are executable via node (shebang), permission bits may vary by OS
  return fs.existsSync(path.join(__dirname, 'test-ablation-study.js'));
});

check('Interpretability test is executable', () => {
  // Files are executable via node (shebang), permission bits may vary by OS
  return fs.existsSync(path.join(__dirname, 'test-interpretability.js'));
});

console.log('\n[10] Test Output Format\n');

check('Tests use consistent output format', () => {
  const moduleTest = fs.readFileSync(path.join(__dirname, 'test-module-validation.js'), 'utf8');
  return moduleTest.includes('='.repeat(40)) && 
         moduleTest.includes('VALIDATION');
});

check('Tests have summary sections', () => {
  const moduleTest = fs.readFileSync(path.join(__dirname, 'test-module-validation.js'), 'utf8');
  return moduleTest.includes('SUMMARY') && 
         moduleTest.includes('Tests Passed') &&
         moduleTest.includes('Success Rate');
});

// ============================================================================
// SUMMARY
// ============================================================================

console.log('\n' + '='.repeat(80));
console.log('VALIDATION FRAMEWORK INTEGRITY CHECK SUMMARY');
console.log('='.repeat(80));
console.log(`\nChecks Passed: ${passed}/${checks}`);
console.log(`Success Rate: ${(passed / checks * 100).toFixed(1)}%\n`);

if (passed === checks) {
  console.log('✅ VALIDATION FRAMEWORK IS FULLY OPERATIONAL');
  console.log('✅ All test files present and properly structured');
  console.log('✅ Documentation complete');
  console.log('✅ Package.json integration verified');
  console.log('✅ Test coverage comprehensive');
  console.log('\nThe validation framework is ready to use!');
  console.log('Run: npm run test-all\n');
} else {
  console.log('⚠️  VALIDATION FRAMEWORK HAS ISSUES');
  console.log(`⚠️  ${checks - passed} check(s) failed`);
  console.log('⚠️  Review and fix the issues above\n');
}

console.log('='.repeat(80));

process.exit(passed === checks ? 0 : 1);