BIT/BoostedInformationTree.py at 1.0 · HephyAnalysisSW/BIT · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
#!/usr/bin/env python
# Standard imports
import cProfile
import sys
import time
import pickle

import numpy as np

from Node import Node

class BoostedInformationTree:

    def __init__( self, training_features, training_weights, training_diff_weights, n_trees = 100, learning_rate = "auto", weights_update_method = "python_loop", calibrated = False, **kwargs ):

        self.n_trees        = n_trees
        self.learning_rate  = learning_rate
        self.calibrate      = calibrated
        # Attempt to learn 98%. (1-learning_rate)^n_trees = 0.02 -> After the fit, the score is at least down to 2%
        if learning_rate == "auto":
            self.learning_rate = 1-0.02**(1./self.n_trees)
        self.kwargs         = kwargs

        self.training_weights       = training_weights
        self.training_diff_weights  = np.copy(training_diff_weights) # Protect the outside from reweighting.
        self.training_features      = training_features

        # how to update the weights in the boosting/learning process
        self.weights_update_method = weights_update_method

        # Will hold the trees
        self.trees                  = []

    @classmethod
    def load(cls, filename):
        old_instance = pickle.load(file( filename ))
        new_instance = cls( None, None, None,
                n_trees = old_instance.n_trees,
                learning_rate = old_instance.learning_rate,
                weights_update_method = old_instance.weights_update_method,
                calibrated = old_instance.calibrated if hasattr(old_instance, "calibrated") else False,
                )
        new_instance.trees = old_instance.trees
        if hasattr( old_instance, "calibration_min_fac" ):
            new_instance.calibration_min_fac = old_instance.calibration_min_fac
        else:
            new_instance.calibration_min_fac = ( 0, 1 )
        return new_instance

    def save(self, filename):
        pickle.dump( self, file( filename, 'w' ) )

    def boost( self ):

        toolbar_width = min(20, self.n_trees)

        # setup toolbar
        sys.stdout.write("[%s]" % (" " * toolbar_width))
        sys.stdout.flush()
        sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '['

        weak_learner_time = 0.0
        update_time = 0.0

        for n_tree in range(self.n_trees):

            # fit to data
            time1 = time.time()
            root = Node(    self.training_features,
                            training_weights        =   self.training_weights,
                            training_diff_weights   =   self.training_diff_weights,
                            **self.kwargs
                        )
            time2 = time.time()
            weak_learner_time += time2 - time1

            # Recall current tree
            self.trees.append( root )

            #print "max/min", max(training_diff_weights), min(training_diff_weights)
            #root.print_tree()
            #histo = score_histo(root)
            # Except for the last node, only take a fraction of the score

            # reduce the score
            time1 = time.time()
            if self.weights_update_method == "python_loop":
                weights_update_delta = np.multiply(self.training_weights, np.array([root.predict(feature) for feature in self.training_features]))
            elif self.weights_update_method == "vectorized":
                weights_update_delta = np.multiply(self.training_weights, root.vectorized_predict(self.training_features, key='score'))
            else:
                raise ValueError("weights update method %s unknown" % self.weights_update_method)
            self.training_diff_weights+= -self.learning_rate*weights_update_delta
            time2 = time.time()
            update_time += time2 - time1

            #np.testing.assert_array_equal(weights_update_delta, weights_update_delta_2, verbose=True)

            # update the bar
            if self.n_trees>=toolbar_width:
                if n_tree % (self.n_trees/toolbar_width)==0:   sys.stdout.write("-")
            sys.stdout.flush()

        sys.stdout.write("]\n") # this ends the progress bar
        print "weak learner time: %.2f" % weak_learner_time
        print "update time: %.2f" % update_time

        self.calibration_min_fac = (0., 1.)
        time1 = time.time()
        if self.calibrate:
            predictions = self.vectorized_predict(self.training_features)
            min_        = np.min(predictions)
            self.calibration_min_fac = ( min_, 1./(np.max(predictions)-min_) )
        time2 = time.time()
        calibration_time = time2 - time1
        print "calibration time: %.2f" % calibration_time

        # purge training data
        del self.training_weights
        del self.training_diff_weights
        del self.training_features

    def predict( self, feature_array, max_n_tree = None, summed = True, last_tree_counts_full = False):
        # list learning rates
        learning_rates = self.learning_rate*np.ones(max_n_tree if max_n_tree is not None else self.n_trees)
        # keep the last tree?
        if last_tree_counts_full and (max_n_tree is None or max_n_tree==self.n_trees):
            learning_rates[-1] = 1

        predictions = np.array([ tree.predict( feature_array ) for tree in self.trees[:max_n_tree] ])

        if summed:
            return ( np.dot(learning_rates, predictions) - self.calibration_min_fac[0])*self.calibration_min_fac[1]
        else:
            return ( learning_rates*predictions - self.calibration_min_fac[0])*self.calibration_min_fac[1]

    def vectorized_predict( self, feature_array, max_n_tree = None, summed = True, last_tree_counts_full = False):
        # list learning rates
        learning_rates = self.learning_rate*np.ones(max_n_tree if max_n_tree is not None else self.n_trees)
        # keep the last tree?
        if last_tree_counts_full and (max_n_tree is None or max_n_tree==self.n_trees):
            learning_rates[-1] = 1

        predictions = np.array([ tree.vectorized_predict( feature_array ) for tree in self.trees[:max_n_tree] ])

        if summed:
            return (np.dot(learning_rates, predictions) - self.calibration_min_fac[0])*self.calibration_min_fac[1]
        else:
            return (learning_rates.reshape(-1, 1)*predictions - self.calibration_min_fac[0])*self.calibration_min_fac[1]