Make active-learning runs reproducible under a fixed seed

libact/base/dataset.py

@@ -6,11 +6,10 @@
 """
 from __future__ import unicode_literals
 
-import random
 import numpy as np
 import scipy.sparse as sp
 
-from libact.utils import zip
+from libact.utils import check_random_state, zip
 
 
 class Dataset(object):
@@ -208,17 +207,25 @@ def get_unlabeled_entries(self):
         """
         return np.where(~self.get_labeled_mask())[0], self._X[~self.get_labeled_mask()]
 
-    def labeled_uniform_sample(self, sample_size, replace=True):
+    def labeled_uniform_sample(self, sample_size, replace=True,
+                               random_state=None):
         """Returns a Dataset object with labeled data only, which is
         resampled uniformly with given sample size.
         Parameter `replace` decides whether sampling with replacement or not.
 
         Parameters
         ----------
         sample_size
+
+        random_state : {int, np.random.RandomState instance, None}, optional (default=None)
+            If int, random_state is passed as parameter to generate
+            np.random.RandomState instance. if np.random.RandomState instance,
+            random_state is the random number generate. If None, the global
+            numpy random state is used, keeping the previous behavior.
         """
-        idx = np.random.choice(np.where(self.get_labeled_mask())[0],
-                               size=sample_size, replace=replace)
+        random_state_ = check_random_state(random_state)
+        idx = random_state_.choice(np.where(self.get_labeled_mask())[0],
+                                   size=sample_size, replace=replace)
         return Dataset(self._X[idx], self._y[idx])
 
 
@@ -229,13 +236,27 @@ def import_libsvm_sparse(filename):
     return Dataset(X.toarray(), y)
 
 
-def import_scipy_mat(filename):
+def import_scipy_mat(filename, random_state=None):
+    """Imports dataset file in scipy mat format and shuffles the entries.
+
+    Parameters
+    ----------
+    filename : str
+        Path to the mat file with 'X' and 'y' entries.
+
+    random_state : {int, np.random.RandomState instance, None}, optional (default=None)
+        If int, random_state is passed as parameter to generate
+        np.random.RandomState instance to shuffle the entries with. if
+        np.random.RandomState instance, random_state is the random number
+        generate. If None, the global numpy random state is used, keeping
+        the previous behavior.
+    """
     from scipy.io import loadmat
     data = loadmat(filename)
     X = data['X']
     y = data['y']
     zipper = list(zip(X, y))
-    np.random.shuffle(zipper)
+    check_random_state(random_state).shuffle(zipper)
     X, y = zip(*zipper)
     X, y = np.array(X), np.array(y).reshape(-1)
     return Dataset(X, y)

libact/base/tests/test_dataset.py

@@ -1,8 +1,10 @@
+import os
+import tempfile
 import unittest
 
 import numpy as np
 
-from libact.base.dataset import Dataset
+from libact.base.dataset import Dataset, import_scipy_mat
 
 
 class TestDatasetMethods(unittest.TestCase):
@@ -106,6 +108,34 @@ def test_labeled_uniform_sample(self):
         with self.assertRaises(ValueError):
             dataset_s = dataset.labeled_uniform_sample(4, replace=False)
 
+    def test_labeled_uniform_sample_random_state(self):
+        dataset = self.setup_dataset()
+        dataset_1 = dataset.labeled_uniform_sample(10, random_state=1126)
+        dataset_2 = dataset.labeled_uniform_sample(10, random_state=1126)
+        dataset_3 = dataset.labeled_uniform_sample(
+            10, random_state=np.random.RandomState(1126))
+        X_1, y_1 = dataset_1.get_entries()
+        X_2, y_2 = dataset_2.get_entries()
+        X_3, y_3 = dataset_3.get_entries()
+        self.assertTrue(np.array_equal(X_1, X_2))
+        self.assertEqual(y_1.tolist(), y_2.tolist())
+        self.assertTrue(np.array_equal(X_1, X_3))
+        self.assertEqual(y_1.tolist(), y_3.tolist())
+
+    def test_import_scipy_mat_random_state(self):
+        from scipy.io import savemat
+        X = np.arange(40).reshape(20, 2)
+        y = np.arange(20).reshape(-1, 1)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            mat_path = os.path.join(tmp_dir, 'dataset.mat')
+            savemat(mat_path, {'X': X, 'y': y})
+            dataset_1 = import_scipy_mat(mat_path, random_state=1126)
+            dataset_2 = import_scipy_mat(mat_path, random_state=1126)
+        X_1, y_1 = dataset_1.get_entries()
+        X_2, y_2 = dataset_2.get_entries()
+        self.assertTrue(np.array_equal(X_1, X_2))
+        self.assertEqual(y_1.tolist(), y_2.tolist())
+
 
 if __name__ == '__main__':
     unittest.main()

libact/labelers/ideal_labeler.py

@@ -5,7 +5,7 @@
 import numpy as np
 
 from libact.base.interfaces import Labeler
-from libact.utils import inherit_docstring_from
+from libact.utils import inherit_docstring_from, check_random_state
 
 
 class IdealLabeler(Labeler):
@@ -18,18 +18,27 @@ class IdealLabeler(Labeler):
     dataset: Dataset object
         Dataset object with the ground-truth label for each sample.
 
+    random_state : {int, np.random.RandomState instance, None}, optional (default=None)
+        If int, random_state is passed as parameter to generate
+        np.random.RandomState instance. if np.random.RandomState instance,
+        random_state is the random number generate. If None, the global
+        numpy random state is used, keeping the previous behavior. Only used
+        to break ties when the queried feature matches multiple samples with
+        different labels.
+
     """
 
-    def __init__(self, dataset, **kwargs):
+    def __init__(self, dataset, random_state=None, **kwargs):
         X, y = dataset.get_entries()
         # make sure the input dataset is fully labeled
         assert (np.array(y) != np.array(None)).all()
         self.X = X
         self.y = y
+        self.random_state_ = check_random_state(random_state)
 
     @inherit_docstring_from(Labeler)
     def label(self, feature):
         yy = self.y[np.where([np.array_equal(x, feature)
                               for x in self.X])[0]]
         ind = np.arange(len(yy))
-        return yy[np.random.choice(ind, 1)[0]]
+        return yy[self.random_state_.choice(ind, 1)[0]]

libact/labelers/tests/test_labelers.py

@@ -39,5 +39,18 @@ def test_mlc_label(self):
         ask_id = lbr.label(np.array([ 6.,  2., 21., 20.,  5.]))
         np.testing.assert_array_equal(ask_id, [0, 0, 1, 0, 1])
 
+    def test_label_random_state(self):
+        """same random_state gives the same label for duplicate features
+        carrying conflicting labels"""
+        X = np.vstack([np.zeros((2, 3)), np.ones((2, 3))])
+        y = np.array([1, 2, 3, 4])
+        dataset = Dataset(X, y)
+        lbr1 = IdealLabeler(dataset, random_state=1126)
+        lbr2 = IdealLabeler(dataset, random_state=1126)
+        labels1 = [lbr1.label(np.zeros(3)) for _ in range(20)]
+        labels2 = [lbr2.label(np.zeros(3)) for _ in range(20)]
+        self.assertEqual(labels1, labels2)
+        self.assertTrue(set(labels1) <= {1, 2})
+
 if __name__ == '__main__':
     unittest.main()

libact/models/sklearn_adapter.py

@@ -4,6 +4,22 @@
 from libact.base.interfaces import Model, ContinuousModel, ProbabilisticModel
 
 
+def _clone_estimator(estimator, random_state=None):
+    """Clone a scikit-learn estimator, optionally overriding its random_state.
+
+    When ``random_state`` is given and the estimator exposes a ``random_state``
+    parameter, the clone is reseeded with it. This lets ensemble strategies
+    give each cloned committee member a distinct, deterministic seed derived
+    from the strategy's own random_state (mirroring scikit-learn's ensemble
+    behavior). Estimators without a ``random_state`` parameter are cloned
+    unchanged.
+    """
+    cloned = clone(estimator)
+    if random_state is not None and 'random_state' in cloned.get_params():
+        cloned.set_params(random_state=random_state)
+    return cloned
+
+
 class SklearnAdapter(Model):
     """Implementation of the scikit-learn classifier to libact model interface.
 
@@ -49,8 +65,9 @@ def score(self, testing_dataset, *args, **kwargs):
         return self._model.score(*(testing_dataset.format_sklearn() + args),
                                 **kwargs)
 
-    def clone(self):
-        return SklearnProbaAdapter(clone(self._model))
+    def clone(self, random_state=None):
+        return SklearnProbaAdapter(
+            _clone_estimator(self._model, random_state))
 
 
 class SklearnProbaAdapter(ProbabilisticModel):
@@ -107,5 +124,6 @@ def predict_real(self, feature, *args, **kwargs):
     def predict_proba(self, feature, *args, **kwargs):
         return self._model.predict_proba(feature, *args, **kwargs)
 
-    def clone(self):
-        return SklearnProbaAdapter(clone(self._model))
+    def clone(self, random_state=None):
+        return SklearnProbaAdapter(
+            _clone_estimator(self._model, random_state))

libact/query_strategies/bald.py

@@ -123,7 +123,19 @@ def __init__(self, dataset, **kwargs):
             if not hasattr(base_model, 'clone'):
                 raise TypeError("base_model must have a 'clone()' method")
             self._base_model = base_model
-            self.models = [base_model.clone() for _ in range(self.n_models)]
+            # Give each cloned committee member a distinct, deterministic seed
+            # derived from random_state_ so a passed random_state fully controls
+            # the ensemble even when the base model is stochastic.
+            clone_seeds = self.random_state_.randint(
+                np.iinfo(np.int32).max, size=self.n_models)
+            self.models = []
+            for seed in clone_seeds:
+                try:
+                    model = base_model.clone(random_state=int(seed))
+                except TypeError:
+                    # Custom clone() implementations may not accept a seed.
+                    model = base_model.clone()
+                self.models.append(model)
         else:
             raise TypeError(
                 "__init__() requires either 'models' or 'base_model' argument"

libact/query_strategies/tests/meson.build

@@ -9,6 +9,7 @@ py_src = [
   'test_information_density.py',
   'test_quire.py',
   'test_realdata.py',
+  'test_seed_reproducibility.py',
   'test_uncertainty_sampling.py',
   'test_variance_reduction.py',
   'utils.py',