Merge pull request #108 from CSSLab/dev

Add new excellent move criteria + move tooltips to Portals

Author: kevinjosethomas

__tests__/types/tree.test.ts

@@ -0,0 +1,177 @@
+import { GameNode } from 'src/types/base/tree'
+import { StockfishEvaluation, MaiaEvaluation } from 'src/types'
+
+describe('GameNode Move Classification', () => {
+  describe('Excellent Move Criteria', () => {
+    it('should classify move as excellent when Maia probability < 10% and winrate is 10% higher than weighted average', () => {
+      const parentNode = new GameNode(
+        'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1',
+      )
+
+      // Mock Stockfish evaluation with winrate vectors
+      const stockfishEval: StockfishEvaluation = {
+        sent: true,
+        depth: 15,
+        model_move: 'e2e4',
+        model_optimal_cp: 50,
+        cp_vec: { e2e4: 50, d2d4: 40, g1f3: 30 },
+        cp_relative_vec: { e2e4: 0, d2d4: -10, g1f3: -20 },
+        winrate_vec: { e2e4: 0.6, d2d4: 0.58, g1f3: 0.4 },
+        winrate_loss_vec: { e2e4: 0, d2d4: -0.02, g1f3: -0.2 },
+      }
+
+      // Mock Maia evaluation with policy probabilities
+      const maiaEval: { [rating: string]: MaiaEvaluation } = {
+        maia_kdd_1500: {
+          policy: {
+            e2e4: 0.5, // 50% probability - most likely move
+            d2d4: 0.3, // 30% probability
+            g1f3: 0.05, // 5% probability - less than 10% threshold
+          },
+          value: 0.6,
+        },
+      }
+
+      // Add analysis to parent node
+      parentNode.addStockfishAnalysis(stockfishEval, 'maia_kdd_1500')
+      parentNode.addMaiaAnalysis(maiaEval, 'maia_kdd_1500')
+
+      // Calculate weighted average manually for verification:
+      // weighted_avg = (0.5 * 0.6 + 0.3 * 0.58 + 0.05 * 0.4) / (0.5 + 0.3 + 0.05)
+      // weighted_avg = (0.3 + 0.174 + 0.02) / 0.85 = 0.494 / 0.85 ≈ 0.581
+      // g1f3 winrate (0.4) is NOT 10% higher than weighted average (0.581)
+      // So g1f3 should NOT be excellent despite low Maia probability
+
+      // Test move with low Maia probability but not high enough winrate
+      const classificationG1f3 = GameNode.classifyMove(
+        parentNode,
+        'g1f3',
+        'maia_kdd_1500',
+      )
+      expect(classificationG1f3.excellent).toBe(false)
+
+      // Now test with a different scenario where a move has both low probability and high winrate
+      const stockfishEval2: StockfishEvaluation = {
+        sent: true,
+        depth: 15,
+        model_move: 'e2e4',
+        model_optimal_cp: 50,
+        cp_vec: { e2e4: 50, d2d4: 40, b1c3: 45 },
+        cp_relative_vec: { e2e4: 0, d2d4: -10, b1c3: -5 },
+        winrate_vec: { e2e4: 0.6, d2d4: 0.45, b1c3: 0.7 },
+        winrate_loss_vec: { e2e4: 0, d2d4: -0.15, b1c3: 0.1 },
+      }
+
+      const maiaEval2: { [rating: string]: MaiaEvaluation } = {
+        maia_kdd_1500: {
+          policy: {
+            e2e4: 0.6, // 60% probability
+            d2d4: 0.35, // 35% probability
+            b1c3: 0.05, // 5% probability - less than 10% threshold
+          },
+          value: 0.6,
+        },
+      }
+
+      const parentNode2 = new GameNode(
+        'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1',
+      )
+      parentNode2.addStockfishAnalysis(stockfishEval2, 'maia_kdd_1500')
+      parentNode2.addMaiaAnalysis(maiaEval2, 'maia_kdd_1500')
+
+      // Calculate weighted average: (0.6 * 0.6 + 0.35 * 0.45 + 0.05 * 0.7) / 1.0
+      // = (0.36 + 0.1575 + 0.035) / 1.0 = 0.5525
+      // b1c3 winrate (0.7) is about 14.75% higher than weighted average (0.5525)
+      // So b1c3 should be excellent (low Maia probability AND high relative winrate)
+
+      const classificationB1c3 = GameNode.classifyMove(
+        parentNode2,
+        'b1c3',
+        'maia_kdd_1500',
+      )
+      expect(classificationB1c3.excellent).toBe(true)
+    })
+
+    it('should not classify move as excellent when Maia probability >= 10%', () => {
+      const parentNode = new GameNode(
+        'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1',
+      )
+
+      const stockfishEval: StockfishEvaluation = {
+        sent: true,
+        depth: 15,
+        model_move: 'e2e4',
+        model_optimal_cp: 50,
+        cp_vec: { e2e4: 50, d2d4: 40 },
+        cp_relative_vec: { e2e4: 0, d2d4: -10 },
+        winrate_vec: { e2e4: 0.6, d2d4: 0.7 },
+        winrate_loss_vec: { e2e4: 0, d2d4: 0.1 },
+      }
+
+      const maiaEval: { [rating: string]: MaiaEvaluation } = {
+        maia_kdd_1500: {
+          policy: {
+            e2e4: 0.8, // 80% probability - above 10% threshold
+            d2d4: 0.2, // 20% probability - above 10% threshold
+          },
+          value: 0.6,
+        },
+      }
+
+      parentNode.addStockfishAnalysis(stockfishEval, 'maia_kdd_1500')
+      parentNode.addMaiaAnalysis(maiaEval, 'maia_kdd_1500')
+
+      // Even though d2d4 has higher winrate than weighted average,
+      // it should not be excellent because Maia probability > 10%
+      const classification = GameNode.classifyMove(
+        parentNode,
+        'd2d4',
+        'maia_kdd_1500',
+      )
+      expect(classification.excellent).toBe(false)
+    })
+
+    it('should not classify move as excellent when winrate advantage < 10%', () => {
+      const parentNode = new GameNode(
+        'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1',
+      )
+
+      const stockfishEval: StockfishEvaluation = {
+        sent: true,
+        depth: 15,
+        model_move: 'e2e4',
+        model_optimal_cp: 50,
+        cp_vec: { e2e4: 50, d2d4: 40, a2a3: 20 },
+        cp_relative_vec: { e2e4: 0, d2d4: -10, a2a3: -30 },
+        winrate_vec: { e2e4: 0.6, d2d4: 0.55, a2a3: 0.62 },
+        winrate_loss_vec: { e2e4: 0, d2d4: -0.05, a2a3: 0.02 },
+      }
+
+      const maiaEval: { [rating: string]: MaiaEvaluation } = {
+        maia_kdd_1500: {
+          policy: {
+            e2e4: 0.7, // 70% probability
+            d2d4: 0.25, // 25% probability
+            a2a3: 0.05, // 5% probability - below 10% threshold
+          },
+          value: 0.6,
+        },
+      }
+
+      parentNode.addStockfishAnalysis(stockfishEval, 'maia_kdd_1500')
+      parentNode.addMaiaAnalysis(maiaEval, 'maia_kdd_1500')
+
+      // Weighted average: (0.7 * 0.6 + 0.25 * 0.55 + 0.05 * 0.62) / 1.0
+      // = (0.42 + 0.1375 + 0.031) / 1.0 = 0.5885
+      // a2a3 winrate (0.62) is only about 3.15% higher than weighted average
+      // So a2a3 should NOT be excellent (advantage < 10%)
+
+      const classification = GameNode.classifyMove(
+        parentNode,
+        'a2a3',
+        'maia_kdd_1500',
+      )
+      expect(classification.excellent).toBe(false)
+    })
+  })
+})

src/components/Analysis/MoveTooltip.tsx

@@ -1,4 +1,5 @@
 import React from 'react'
+import { createPortal } from 'react-dom'
 import { ColorSanMapping } from 'src/types'
 
 interface MoveTooltipProps {
@@ -22,14 +23,14 @@ export const MoveTooltip: React.FC<MoveTooltipProps> = ({
   position,
   isVisible = true,
 }) => {
-  if (!isVisible || !position) return null
+  if (!isVisible || !position || typeof window === 'undefined') return null
 
   const san = colorSanMapping[move]?.san ?? move
   const color = colorSanMapping[move]?.color ?? '#fff'
 
-  return (
+  const tooltipContent = (
     <div
-      className="pointer-events-none fixed z-50 flex w-auto min-w-[12rem] flex-col overflow-hidden rounded-lg border border-white/30 bg-background-1 backdrop-blur-sm"
+      className="pointer-events-none fixed z-50 flex w-auto min-w-[12rem] flex-col overflow-hidden rounded-lg border border-white/30 bg-background-1 text-primary backdrop-blur-sm"
       style={{
         left: position.x + 15,
         top: position.y - 10,
@@ -83,4 +84,6 @@ export const MoveTooltip: React.FC<MoveTooltipProps> = ({
       </div>
     </div>
   )
+
+  return createPortal(tooltipContent, document.body)
 }

src/constants/analysis.ts

@@ -3,16 +3,16 @@
  *
  * 1. BLUNDER (??): More than 10% Stockfish winrate loss compared to best move
  * 2. INACCURACY (?!): More than 5% Stockfish winrate loss compared to best move
- * 3. EXCELLENT (!!): Less than 2% Stockfish winrate loss AND less than 10% Maia probability
+ * 3. EXCELLENT (!!): At least 10% higher Stockfish winrate than weighted average AND less than 10% Maia probability
  *
- * Note: All winrate loss comparisons are against the BEST POSSIBLE MOVE in the
- * same position, NOT the previous position.
+ * Note: Blunder and inaccuracy winrate loss comparisons are against the BEST POSSIBLE MOVE.
+ * Excellent moves are compared against the weighted average winrate (using Maia probabilities as weights).
  */
 
 export const MOVE_CLASSIFICATION_THRESHOLDS = {
   BLUNDER_THRESHOLD: 0.1,
   INACCURACY_THRESHOLD: 0.05,
-  EXCELLENT_WINRATE_THRESHOLD: 0.02,
+  EXCELLENT_WINRATE_ADVANTAGE_THRESHOLD: 0.05,
   MAIA_UNLIKELY_THRESHOLD: 0.1,
   GOOD_THRESHOLD: -0.05,
 } as const

src/types/base/tree.ts

@@ -302,22 +302,6 @@ export class GameNode {
           !blunder &&
           absoluteWinrateLoss >=
             MOVE_CLASSIFICATION_THRESHOLDS.INACCURACY_THRESHOLD
-
-        // Excellent: Less than 2% winrate loss AND less than 10% Maia probability
-        if (
-          absoluteWinrateLoss <
-          MOVE_CLASSIFICATION_THRESHOLDS.EXCELLENT_WINRATE_THRESHOLD
-        ) {
-          if (maiaEval && activeModel && maiaEval[activeModel]) {
-            const policy = maiaEval[activeModel].policy
-            if (policy && move in policy) {
-              const probability = policy[move]
-              excellent =
-                probability <=
-                MOVE_CLASSIFICATION_THRESHOLDS.MAIA_UNLIKELY_THRESHOLD
-            }
-          }
-        }
       }
     } else {
       // Fallback to centipawn-based classification if winrate not available
@@ -327,6 +311,44 @@ export class GameNode {
       }
     }
 
+    // Excellent move criteria: Less than 10% Maia probability AND
+    // at least 10% higher winrate than weighted average
+    if (
+      maiaEval &&
+      activeModel &&
+      maiaEval[activeModel] &&
+      stockfishEval.winrate_vec
+    ) {
+      const policy = maiaEval[activeModel].policy
+      if (policy && move in policy) {
+        const probability = policy[move]
+
+        // Check Maia probability threshold
+        const lowMaiaProbability =
+          probability <= MOVE_CLASSIFICATION_THRESHOLDS.MAIA_UNLIKELY_THRESHOLD
+
+        if (lowMaiaProbability) {
+          // Calculate weighted average winrate using Maia probabilities as weights
+          const weightedAverageWinrate = this.calculateWeightedAverageWinrate(
+            stockfishEval.winrate_vec,
+            policy,
+          )
+
+          const currentMoveWinrate = stockfishEval.winrate_vec[move]
+          if (
+            currentMoveWinrate !== undefined &&
+            weightedAverageWinrate !== null
+          ) {
+            // Check if current move is at least x% higher than weighted average
+            excellent =
+              currentMoveWinrate >=
+              weightedAverageWinrate +
+                MOVE_CLASSIFICATION_THRESHOLDS.EXCELLENT_WINRATE_ADVANTAGE_THRESHOLD
+          }
+        }
+      }
+    }
+
     return {
       blunder,
       inaccuracy,
@@ -335,6 +357,29 @@ export class GameNode {
     }
   }
 
+  // Helper method to calculate weighted average winrate
+  private calculateWeightedAverageWinrate(
+    winrateVec: { [move: string]: number },
+    maiaPolicy: { [move: string]: number },
+  ): number | null {
+    let totalWeight = 0
+    let weightedSum = 0
+
+    // Calculate weighted sum using Maia probabilities as weights
+    for (const move in maiaPolicy) {
+      const weight = maiaPolicy[move]
+      const winrate = winrateVec[move]
+
+      if (weight !== undefined && winrate !== undefined) {
+        weightedSum += weight * winrate
+        totalWeight += weight
+      }
+    }
+
+    // Return weighted average, or null if no valid data
+    return totalWeight > 0 ? weightedSum / totalWeight : null
+  }
+
   private classifyMoveByWinrate(
     node: GameNode,
     move: string,