feat: Add full evaluation type for regression checks and baseline saving, and update CI workflow to use it.

Author: ronibhakta1

.github/workflows/llm-evals.yml

@@ -149,12 +149,12 @@ jobs:
 
       - name: Check for regressions
         id: regression_check
-        run: npm run eval:check-regressions
+        run: npm run eval:check-regressions -- --type=full
         continue-on-error: true
 
       - name: Save as new baseline
         if: github.ref == 'refs/heads/main' && steps.regression_check.outcome == 'success'
-        run: npm run eval:baseline
+        run: npm run eval:baseline -- --type=full
 
       - name: Upload eval results
         uses: actions/upload-artifact@v4

README.md

@@ -3,7 +3,7 @@
 [![npm version](https://img.shields.io/npm/v/zon-format.svg)](https://www.npmjs.com/package/zon-format)
 [![GitHub stars](https://img.shields.io/github/stars/ZON-Format/zon-TS?style=social)](https://github.com/ZON-Format/zon-TS)
 [![TypeScript](https://img.shields.io/badge/TypeScript-5.x-blue.svg)](https://www.typescriptlang.org/)
-[![Tests](https://img.shields.io/badge/tests-288%2F288%20passing-brightgreen.svg)](#quality--testing)
+[![Tests](https://img.shields.io/badge/tests-297%2F297  %20passing-brightgreen.svg)](#quality--testing)
 [![npm downloads](https://img.shields.io/npm/dm/zon-format?color=red)](https://www.npmjs.com/package/zon-format)
 [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
 

benchmarks/scripts/check-regressions.js

@@ -9,16 +9,78 @@ const fs = require('fs');
 const path = require('path');
 const { ZonEvaluator, registerBuiltinMetrics, FileEvalStorage } = require('../../dist/evals');
 
+const { parseArgs } = require('util');
+
 async function checkRegressions() {
-  console.log('🔍 Checking for regressions...\n');
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      type: { type: 'string', default: 'smoke' }
+    }
+  });
+
+  console.log(`🔍 Checking for regressions for ${values.type}...\n`);
 
+  if (values.type === 'full') {
+    const resultsPath = path.join(__dirname, '../results/accuracy-results.json');
+    const baselinePath = path.join(__dirname, '../results/accuracy-baseline.json');
+    
+    if (!fs.existsSync(resultsPath)) {
+      console.log('⚠️  No full evaluation results found');
+      process.exit(0);
+    }
+    
+    if (!fs.existsSync(baselinePath)) {
+      console.log('ℹ️  No baseline found - this will become the baseline');
+      process.exit(0);
+    }
+    
+    const results = JSON.parse(fs.readFileSync(resultsPath, 'utf-8'));
+    const baseline = JSON.parse(fs.readFileSync(baselinePath, 'utf-8'));
+    
+    // Compare accuracy and efficiency
+    let hasCritical = false;
+    const models = Object.keys(results.models);
+    
+    models.forEach(model => {
+      const currentModel = results.models[model];
+      const baselineModel = baseline.models[model];
+      
+      if (!baselineModel) return;
+      
+      Object.keys(currentModel).forEach(format => {
+        const current = currentModel[format];
+        const base = baselineModel[format];
+        
+        if (!base) return;
+        
+        // Check accuracy regression
+        const currentAcc = current.total > 0 ? (current.correct / current.total) : 0;
+        const baseAcc = base.total > 0 ? (base.correct / base.total) : 0;
+        
+        if (currentAcc < baseAcc - 0.05) { // 5% drop
+           console.log(`🔴 ${model}/${format} Accuracy: ${baseAcc.toFixed(2)} -> ${currentAcc.toFixed(2)}`);
+           hasCritical = true;
+        }
+      });
+    });
+    
+    if (hasCritical) {
+      console.error('❌ Critical regressions detected');
+      process.exit(1);
+    }
+    
+    console.log('✅ No regressions detected!\n');
+    process.exit(0);
+  }
+
   const storage = new FileEvalStorage('./benchmarks/results');
 
   try {
     // Load latest results
     const latest = await storage.getLatest('smoke-test');
     if (!latest) {
-      console.log('⚠️  No recent eval results found');
+      console.log('⚠️  No recent smoke test results found');
       process.exit(0);
     }
 

benchmarks/scripts/save-baseline.js

@@ -9,16 +9,43 @@ const fs = require('fs');
 const path = require('path');
 const { FileEvalStorage } = require('../../dist/evals');
 
+const { parseArgs } = require('util');
+
 async function saveBaseline() {
-  console.log('💾 Saving baseline...\n');
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      type: { type: 'string', default: 'smoke' }
+    }
+  });
+
+  console.log(`💾 Saving baseline for ${values.type}...\n`);
 
+  if (values.type === 'full') {
+    const resultsPath = path.join(__dirname, '../results/accuracy-results.json');
+    const baselinePath = path.join(__dirname, '../results/accuracy-baseline.json');
+    
+    if (!fs.existsSync(resultsPath)) {
+      console.error('❌ No full evaluation results found');
+      process.exit(1);
+    }
+    
+    const results = JSON.parse(fs.readFileSync(resultsPath, 'utf-8'));
+    fs.writeFileSync(baselinePath, JSON.stringify(results, null, 2));
+    
+    console.log('✅ Full baseline saved successfully!');
+    console.log(`   Timestamp: ${results.timestamp}`);
+    process.exit(0);
+  }
+
   const storage = new FileEvalStorage('./benchmarks/results');
 
   try {
     // Load latest results
     const latest = await storage.getLatest('smoke-test');
     if (!latest) {
-      console.error('❌ No recent eval results to save as baseline');
+      console.error('❌ No recent smoke test results to save as baseline');
+      console.error('   Run "npm run eval:smoke" first');
       process.exit(1);
     }
 
@@ -31,7 +58,7 @@ async function saveBaseline() {
 
     await storage.save(baselineResult);
 
-    console.log('✅ Baseline saved successfully!');
+    console.log('✅ Smoke test baseline saved successfully!');
     console.log(`   Test ID: ${baselineResult.testId}`);
     console.log(`   Timestamp: ${new Date(baselineResult.timestamp).toISOString()}`);
     process.exit(0);