fix(select): Fix bug in the way the UCB1 was calculated on state brought in from the Transposition t

When calculating the UCB1 for a node, the current algorithm uses the parent node's visit count as a
proxy for the total of the siblings' visit count. This creates a problem when an existing state is
brought in from the transposition table. At that moment, the parent's visit count is no longer
reprensentatiove of the sum of its children visit count. Fix the algorithm so that it uses the
correct number.

Author: Philippe Vaillancourt

src/controller.ts

@@ -9,7 +9,7 @@ import {
 import { MCTSFacade, DefaultMCTSFacade } from './mcts/mcts'
 import { DataStore } from './data-store'
 import { DefaultSelect } from './mcts/select/select'
-import { DefaultExpand } from './mcts/expand/expand'
+import { DefaultExpand, Expand } from './mcts/expand/expand'
 import { UCB1, DefaultUCB1, DefaultBestChild } from './mcts/select/best-child/best-child'
 import {
   DefaultSimulate,

src/mcts/expand/expand.ts

@@ -20,9 +20,6 @@ export interface Expand<State, Action> {
  *
  * @hidden
  * @internal
- * @export
- * @class DefaultExpand
- * @implements {Expand<State, Action>}
  * @template State
  * @template Action
  */

src/mcts/select/best-child/best-child.ts

@@ -44,10 +44,10 @@ export class DefaultBestChild<State extends Playerwise, Action>
     if (!node.children.length) {
       return undefined
     }
-
+    const sumChildVisits = node.children.reduce((p, c) => p + c.mctsState.visits, 0)
     const selectedNode = node.children.reduce((p, c) => {
-      return this.UCB1_.run(node.mctsState, p.mctsState, exploit) >
-        this.UCB1_.run(node.mctsState, c.mctsState, exploit)
+      return this.UCB1_.run(sumChildVisits, p.mctsState, exploit) >
+        this.UCB1_.run(sumChildVisits, c.mctsState, exploit)
         ? p
         : c
     })
@@ -66,7 +66,7 @@ export class DefaultBestChild<State extends Playerwise, Action>
  * @template Action
  */
 export interface UCB1<State, Action> {
-  run(parent: MCTSState<State, Action>, child: MCTSState<State, Action>, exploit?: boolean): number
+  run(sumChildVisits: number, child: MCTSState<State, Action>, exploit?: boolean): number
 }
 
 /**
@@ -90,10 +90,10 @@ export class DefaultUCB1<State, Action> implements UCB1<State, Action> {
    * @returns {number}
    * @memberof DefaultUCB1
    */
-  run(parent: MCTSState<State, Action>, child: MCTSState<State, Action>, exploit = false): number {
+  run(sumChildVisits: number, child: MCTSState<State, Action>, exploit = false): number {
     if (exploit) this.explorationParam_ = 0
     const exploitationTerm = child.reward / child.visits
-    const explorationTerm = Math.sqrt(Math.log(parent.visits) / child.visits)
+    const explorationTerm = Math.sqrt(Math.log(sumChildVisits) / child.visits)
     return exploitationTerm + this.explorationParam_ * explorationTerm
   }
 }

src/mcts/select/select.ts

@@ -41,7 +41,8 @@ export class DefaultSelect<State extends Playerwise, Action> implements Select<S
       const child = this.bestChild_.run(node)
       if (!child) return this.expand_.run(node)
       if (node.isNotFullyExpanded()) {
-        const ucb1 = this.ucb1_.run(node.mctsState, child.mctsState)
+        const sumChildVisits = node.children.reduce((p, c) => p + c.mctsState.visits, 0)
+        const ucb1 = this.ucb1_.run(sumChildVisits, child.mctsState)
         if (ucb1 < this.fpuParam_) {
           return this.expand_.run(node)
         }

test/mcts.test.ts

@@ -98,12 +98,11 @@ describe('The DefaultUCB1 function', () => {
         board: ticTacToeBoard,
         player: 1
       }
-      const parent = new MCTSState(state)
+
       const child = new MCTSState(state)
-      parent.visits = 300
       child.visits = 100
       child.reward = 50
-      expect(ucb1.run(parent, child)).toBeCloseTo(0.8377)
+      expect(ucb1.run(300, child)).toBeCloseTo(0.8377)
     })
   })
 })