add prediction param for care scores

Author: willwade

scripts/analysis/compare_comprehensive.ts

@@ -92,6 +92,7 @@ function printScenarioDetails(comp: any, name1: string, name2: string) {
 
   // CARE Components
   const care = comp.care_components;
+  console.log(`${'CARE Score (Composite)'.padEnd(25)} | ${care.care_score.toFixed(2).padEnd(25)} | ${care.comp_care_score.toFixed(2).padEnd(25)}`);
   console.log(`${'CARE Sentence Effort'.padEnd(25)} | ${care.sentences.toFixed(3).padEnd(25)} | ${care.comp_sentences.toFixed(3).padEnd(25)}`);
   console.log(`${'Core Vocabulary Found'.padEnd(25)} | ${care.core.toString().padEnd(25)} | ${care.comp_core.toString().padEnd(25)}`);
   console.log(`${'Fringe Vocabulary Found'.padEnd(25)} | ${care.fringe.toString().padEnd(25)} | ${care.comp_fringe.toString().padEnd(25)}`);

scripts/analysis/compare_pagesets.ts

@@ -7,18 +7,28 @@ import * as path from 'path';
 async function main() {
   const args = process.argv.slice(2);
   if (args.length < 2) {
-    console.log('Usage: npx ts-node compare_pagesets.ts <file1> <file2> [--spelling1 <id>] [--spelling2 <id>]');
+    console.log('Usage: npx ts-node compare_pagesets.ts <file1> <file2> [--spelling1 <id>] [--spelling2 <id>] [--use-prediction] [--prediction-selections <n>]');
+    console.log('');
+    console.log('Options:');
+    console.log('  --use-prediction          Use prediction instead of common word efforts for missing words');
+    console.log('  --prediction-selections  Average number of prediction selections (default: 1.5)');
+    console.log('');
+    console.log('Note: By default, missing words use common word baseline efforts (matching Ruby aac-metrics)');
     process.exit(1);
   }
 
   const file1 = args[0];
   const file2 = args[1];
   let spelling1: string | undefined;
   let spelling2: string | undefined;
+  let usePrediction = false;
+  let predictionSelections = 1.5;
 
   for (let i = 0; i < args.length; i++) {
     if (args[i] === '--spelling1') spelling1 = args[i + 1];
     if (args[i] === '--spelling2') spelling2 = args[i + 1];
+    if (args[i] === '--use-prediction') usePrediction = true;
+    if (args[i] === '--prediction-selections') predictionSelections = parseFloat(args[i + 1]);
   }
 
   const calculator = new Analytics.MetricsCalculator();
@@ -27,7 +37,8 @@ async function main() {
   console.log(`\n📚 AAC Pageset Comparison`);
   console.log(`=========================`);
   console.log(`Target 1: ${path.basename(file1)}`);
-  console.log(`Target 2: ${path.basename(file2)}\n`);
+  console.log(`Target 2: ${path.basename(file2)}`);
+  console.log(`Missing word method: ${usePrediction ? 'Prediction' : 'Common word efforts'}${usePrediction ? ` (selections: ${predictionSelections})` : ''}\n`);
 
   try {
     // Load and analyze 1
@@ -40,7 +51,11 @@ async function main() {
     const tree2 = p2.loadIntoTree(path.resolve(process.cwd(), file2));
     const metrics2 = calculator.analyze(tree2, { spellingPageId: spelling2 });
 
-    const comparison = comparer.compare(metrics1, metrics2, { includeSentences: true });
+    const comparison = comparer.compare(metrics1, metrics2, {
+      includeSentences: true,
+      usePrediction,
+      predictionSelections
+    });
 
     printDetailedComparison(comparison, path.basename(file1), path.basename(file2));
 
@@ -97,6 +112,11 @@ function printDetailedComparison(comp: any, name1: string, name2: string) {
   console.log(`Avg Sentence Effort:  ${care.sentences.toFixed(3)} (T1) vs ${care.comp_sentences.toFixed(3)} (T2)`);
   console.log(`Fringe Words Found:   ${care.fringe} (T1) vs ${care.comp_fringe} (T2)`);
   console.log(line);
+  console.log(`\n📈 CARE SCORE (Composite)`);
+  console.log(line);
+  console.log(`T1 Score: ${care.care_score.toFixed(2)} (higher is better)`);
+  console.log(`T2 Score: ${care.comp_care_score.toFixed(2)} (higher is better)`);
+  console.log(line);
   console.log(`\n`);
 }
 

src/utilities/analytics/metrics/comparison.ts

@@ -9,6 +9,8 @@ import { MetricsResult, ButtonMetrics, ComparisonResult } from './types';
 import { SentenceAnalyzer } from './sentence';
 import { VocabularyAnalyzer } from './vocabulary';
 import { ReferenceLoader } from '../reference/index';
+import { spellingEffort, predictionEffort } from './effort';
+import { MetricsOptions } from './types';
 
 export class ComparisonAnalyzer {
   private vocabAnalyzer: VocabularyAnalyzer;
@@ -37,7 +39,7 @@ export class ComparisonAnalyzer {
     options?: {
       includeSentences?: boolean;
       locale?: string;
-    }
+    } & Partial<MetricsOptions>
   ): ComparisonResult {
     // Create base result from target
     const baseResult = { ...targetResult };
@@ -102,7 +104,8 @@ export class ComparisonAnalyzer {
     const careComponents = this.calculateCareComponents(
       targetResult,
       compareResult,
-      overlappingWords
+      overlappingWords,
+      options
     );
 
     // Analyze high/low effort words
@@ -269,8 +272,61 @@ export class ComparisonAnalyzer {
   private calculateCareComponents(
     targetResult: MetricsResult,
     compareResult: MetricsResult,
-    _overlappingWords: string[]
+    _overlappingWords: string[],
+    options?: {
+      includeSentences?: boolean;
+      locale?: string;
+    } & Partial<MetricsOptions>
   ): ComparisonResult['care_components'] {
+    // Load common words with baseline efforts (matching Ruby line 527-534)
+    const commonWordsData = this.referenceLoader.loadCommonWords();
+    const commonWords = new Map<string, number>();
+    commonWordsData.words.forEach((word: string) => {
+      commonWords.set(word.toLowerCase(), commonWordsData.efforts[word] || 0);
+    });
+
+    // Determine prediction settings (default: use common words efforts, not prediction)
+    const usePrediction = options?.usePrediction || false; // Default FALSE (use common words)
+    const predictionSelections = options?.predictionSelections || 1.5;
+    const debugMode = process.env.DEBUG_METRICS === 'true';
+
+    // Helper function to calculate fallback effort
+    const getFallbackEffort = (
+      word: string,
+      hasPrediction: boolean,
+      spellingBaseEffort?: number
+    ): number => {
+      const wordLower = word.toLowerCase();
+
+      // Check common words efforts first (matching Ruby line 533)
+      if (commonWords.has(wordLower)) {
+        const effort = commonWords.get(wordLower);
+        return effort !== undefined ? effort : spellingEffort(word, 10, 2.5);
+      }
+
+      // If usePrediction is true and prediction is available, use prediction
+      if (usePrediction && hasPrediction && spellingBaseEffort !== undefined) {
+        return predictionEffort(spellingBaseEffort, 2.5, predictionSelections, 2);
+      }
+
+      // Fallback to manual spelling (matching Ruby spelling_effort: 10 + word.length * 2.5)
+      return spellingEffort(word, 10, 2.5);
+    };
+
+    // Debug: Check settings
+    const targetHasPrediction =
+      targetResult.has_dynamic_prediction && targetResult.spelling_effort_base !== undefined;
+    const _compareHasPrediction =
+      compareResult.has_dynamic_prediction && compareResult.spelling_effort_base !== undefined;
+    if (debugMode) {
+      console.log(`\n🔍 DEBUG Fallback Effort Settings:`);
+      console.log(`  Common words loaded: ${commonWords.size}`);
+      console.log(`  usePrediction option: ${usePrediction}`);
+      console.log(`  Target has prediction capability: ${targetHasPrediction}`);
+      console.log(
+        `  Target spelling_base: ${targetResult.spelling_effort_base?.toFixed(2) || 'undefined'}`
+      );
+    }
     // Create word maps with normalized keys
     const targetWords = new Map<string, ButtonMetrics>();
     targetResult.buttons.forEach((btn) => {
@@ -293,68 +349,200 @@ export class ComparisonAnalyzer {
     // Load reference data
     const coreLists = this.referenceLoader.loadCoreLists();
     const fringe = this.referenceLoader.loadFringe();
+    const commonFringe = this.referenceLoader.loadCommonFringe();
     const sentences = this.referenceLoader.loadSentences();
 
-    // Calculate core coverage
+    // Calculate core coverage and effort (matching Ruby lines 609-647)
     let coreCount = 0;
     let compCoreCount = 0;
+    let targetCoreEffort = 0;
+    let compCoreEffort = 0;
     const allCoreWords = new Set<string>();
     coreLists.forEach((list) => {
       list.words.forEach((word) => allCoreWords.add(word.toLowerCase()));
     });
 
     allCoreWords.forEach((word) => {
       const key = this.normalize(word);
-      if (targetWords.has(key)) coreCount++;
-      if (compareWords.has(key)) compCoreCount++;
+      const targetBtn = targetWords.get(key);
+      const compareBtn = compareWords.get(key);
+
+      if (targetBtn) {
+        coreCount++;
+        targetCoreEffort += targetBtn.effort;
+      } else {
+        // Fallback to spelling or prediction effort
+        targetCoreEffort += getFallbackEffort(
+          word,
+          targetResult.has_dynamic_prediction || false,
+          targetResult.spelling_effort_base
+        );
+      }
+
+      if (compareBtn) {
+        compCoreCount++;
+        compCoreEffort += compareBtn.effort;
+      } else {
+        compCoreEffort += getFallbackEffort(
+          word,
+          compareResult.has_dynamic_prediction || false,
+          compareResult.spelling_effort_base
+        );
+      }
     });
 
-    // Calculate sentence construction effort
-    let sentenceEffort = 0;
-    let compSentenceEffort = 0;
-    let sentenceWordCount = 0;
+    const avgCoreEffort = allCoreWords.size > 0 ? targetCoreEffort / allCoreWords.size : 0;
+    const avgCompCoreEffort = allCoreWords.size > 0 ? compCoreEffort / allCoreWords.size : 0;
+
+    // Calculate core component scores (matching Ruby lines 644-647)
+    const coreScore = avgCoreEffort * 5.0;
+    const compCoreScore = avgCompCoreEffort * 5.0;
+
+    // Calculate sentence construction effort (matching Ruby lines 654-668)
+    const sentenceEfforts: number[] = [];
+    const compSentenceEfforts: number[] = [];
 
     sentences.forEach((words) => {
+      let targetSentenceEffort = 0;
+      let compSentenceEffort = 0;
+
       words.forEach((word) => {
         const key = this.normalize(word);
         const targetBtn = targetWords.get(key);
         const compareBtn = compareWords.get(key);
 
         if (targetBtn) {
-          sentenceEffort += targetBtn.effort;
+          targetSentenceEffort += targetBtn.effort;
         } else {
-          sentenceEffort += 10 + word.length * 2.5; // Spelling effort
+          targetSentenceEffort += getFallbackEffort(
+            word,
+            targetResult.has_dynamic_prediction || false,
+            targetResult.spelling_effort_base
+          );
         }
 
         if (compareBtn) {
           compSentenceEffort += compareBtn.effort;
         } else {
-          compSentenceEffort += 10 + word.length * 2.5;
+          compSentenceEffort += getFallbackEffort(
+            word,
+            compareResult.has_dynamic_prediction || false,
+            compareResult.spelling_effort_base
+          );
         }
-
-        sentenceWordCount++;
       });
+
+      // Average effort per sentence (matching Ruby line 657)
+      sentenceEfforts.push(targetSentenceEffort / words.length);
+      compSentenceEfforts.push(compSentenceEffort / words.length);
     });
 
-    const avgSentenceEffort = sentenceWordCount > 0 ? sentenceEffort / sentenceWordCount : 0;
+    const avgSentenceEffort =
+      sentenceEfforts.length > 0
+        ? sentenceEfforts.reduce((a, b) => a + b, 0) / sentenceEfforts.length
+        : 0;
     const compAvgSentenceEffort =
-      sentenceWordCount > 0 ? compSentenceEffort / sentenceWordCount : 0;
+      compSentenceEfforts.length > 0
+        ? compSentenceEfforts.reduce((a, b) => a + b, 0) / compSentenceEfforts.length
+        : 0;
+
+    // Sentence component scores (matching Ruby line 665-668)
+    const sentenceScore = avgSentenceEffort * 3.0;
+    const compSentenceScore = compAvgSentenceEffort * 3.0;
 
-    // Calculate fringe coverage
+    // Calculate fringe effort (matching Ruby lines 670-687)
+    const fringeEfforts: number[] = [];
+    const compFringeEfforts: number[] = [];
     let fringeCount = 0;
     let compFringeCount = 0;
+
+    fringe.forEach((word: string) => {
+      const key = this.normalize(word);
+      const targetBtn = targetWords.get(key);
+      const compareBtn = compareWords.get(key);
+
+      if (targetBtn) {
+        fringeEfforts.push(targetBtn.effort);
+        fringeCount++;
+      } else {
+        fringeEfforts.push(
+          getFallbackEffort(
+            word,
+            targetResult.has_dynamic_prediction || false,
+            targetResult.spelling_effort_base
+          )
+        );
+      }
+
+      if (compareBtn) {
+        compFringeEfforts.push(compareBtn.effort);
+        compFringeCount++;
+      } else {
+        compFringeEfforts.push(
+          getFallbackEffort(
+            word,
+            compareResult.has_dynamic_prediction || false,
+            compareResult.spelling_effort_base
+          )
+        );
+      }
+    });
+
+    const avgFringeEffort =
+      fringeEfforts.length > 0
+        ? fringeEfforts.reduce((a, b) => a + b, 0) / fringeEfforts.length
+        : 0;
+    const avgCompFringeEffort =
+      compFringeEfforts.length > 0
+        ? compFringeEfforts.reduce((a, b) => a + b, 0) / compFringeEfforts.length
+        : 0;
+
+    // Fringe component scores (matching Ruby line 684-687)
+    const fringeScore = avgFringeEffort * 2.0;
+    const compFringeScore = avgCompFringeEffort * 2.0;
+
+    // Calculate common fringe effort (matching Ruby lines 689-705)
+    const commonFringeEfforts: number[] = [];
+    const compCommonFringeEfforts: number[] = [];
     let commonFringeCount = 0;
 
-    fringe.forEach((word) => {
+    commonFringe.forEach((word: string) => {
       const key = this.normalize(word);
-      const inTarget = targetWords.has(key);
-      const inCompare = compareWords.has(key);
+      const targetBtn = targetWords.get(key);
+      const compareBtn = compareWords.get(key);
 
-      if (inTarget) fringeCount++;
-      if (inCompare) compFringeCount++;
-      if (inTarget && inCompare) commonFringeCount++;
+      if (targetBtn && compareBtn) {
+        commonFringeEfforts.push(targetBtn.effort);
+        compCommonFringeEfforts.push(compareBtn.effort);
+        commonFringeCount++;
+      }
     });
 
+    const avgCommonFringeEffort =
+      commonFringeEfforts.length > 0
+        ? commonFringeEfforts.reduce((a, b) => a + b, 0) / commonFringeEfforts.length
+        : 0;
+    const avgCompCommonFringeEffort =
+      compCommonFringeEfforts.length > 0
+        ? compCommonFringeEfforts.reduce((a, b) => a + b, 0) / compCommonFringeEfforts.length
+        : 0;
+
+    // Common fringe component scores (matching Ruby line 702-705)
+    const commonFringeScore = avgCommonFringeEffort * 1.0;
+    const compCommonFringeScore = avgCompCommonFringeEffort * 1.0;
+
+    // Calculate total CARE effort tally (matching Ruby lines 707-708)
+    const PLACEHOLDER = 70;
+    const targetEffortTally =
+      coreScore + sentenceScore + fringeScore + commonFringeScore + PLACEHOLDER;
+    const compEffortTally =
+      compCoreScore + compSentenceScore + compFringeScore + compCommonFringeScore + PLACEHOLDER;
+
+    // Calculate final CARE scores (matching Ruby line 710-711)
+    // res[:target_effort_score] = [0.0, 350.0 - target_effort_tally].max
+    const careScore = Math.max(0, 350.0 - targetEffortTally);
+    const compCareScore = Math.max(0, 350.0 - compEffortTally);
+
     return {
       core: coreCount,
       comp_core: compCoreCount,
@@ -364,6 +552,9 @@ export class ComparisonAnalyzer {
       comp_fringe: compFringeCount,
       common_fringe: commonFringeCount,
       comp_common_fringe: commonFringeCount,
+      // New composite CARE scores
+      care_score: careScore,
+      comp_care_score: compCareScore,
     };
   }