Support hyper-parameter search for next-item recommenders (#643) (#702)

* Support hyper-parameter search for next-item recommenders (#643)

GridSearch/RandomSearch could not tune NextItemRecommender models
evaluated with NextItemEvaluation:
  - NextItemEvaluation.evaluate() rejected the search wrapper because
    it is a Recommender, not a NextItemRecommender.
  - BaseSearch.fit() scored next-item models with the standard
    ranking_eval, whose rank()/score() path is incompatible with the
    session-based score(history_items=...) signature.

Fix:
  - Accept a search wrapper whose .model is a NextItemRecommender.
  - Route next-item models through next_item_evaluation.ranking_eval
    during search, using the eval_method's exclude_unknowns/mode.
  - Delegate transform/score/rank from BaseSearch to the best model
    so the fitted wrapper evaluates transparently.

Add GridSearch/RandomSearch next-item tests.

* update get model seed

* add seed for reproducible hyperopt search

---------

Co-authored-by: hieuddo <hieu.dd.1998@gmail.com>

Author: nsiddharth

cornac/eval_methods/next_item_evaluation.py

@@ -335,7 +335,10 @@ def evaluate(self, model, metrics, user_based, show_validation=True):
         -------
         res: :obj:`cornac.experiment.Result`
         """
-        if not isinstance(model, NextItemRecommender):
+        base_model = getattr(model, "model", None)
+        if not isinstance(model, NextItemRecommender) and not isinstance(
+            base_model, NextItemRecommender
+        ):
             raise ValueError("model must be a NextItemRecommender but '%s' is provided" % type(model))
 
         if self.train_set is None:

cornac/hyperopt.py

@@ -17,9 +17,10 @@
 import numpy as np
 from itertools import product
 
-from .models import Recommender
+from .models import Recommender, NextItemRecommender
 from .metrics import RatingMetric, RankingMetric
 from .eval_methods import rating_eval, ranking_eval
+from .eval_methods.next_item_evaluation import ranking_eval as next_item_ranking_eval
 from .utils import get_rng
 
 
@@ -147,6 +148,16 @@ def fit(self, train_set, val_set=None):
 
             if isinstance(self.metric, RatingMetric):
                 score = rating_eval(model, [self.metric], val_set)[0][0]
+            elif isinstance(model, NextItemRecommender):
+                score = next_item_ranking_eval(
+                    model,
+                    [self.metric],
+                    train_set,
+                    val_set,
+                    exclude_unknowns=self.eval_method.exclude_unknowns,
+                    mode=self.eval_method.mode,
+                    verbose=False,
+                )[0][0]
             else:
                 score = ranking_eval(
                     model,
@@ -171,9 +182,17 @@ def fit(self, train_set, val_set=None):
 
         return self
 
-    def score(self, user_idx, item_idx=None):
+    def transform(self, test_set):
+        """Delegate test-set transformation to the best searched model."""
+        return self.best_model.transform(test_set)
+
+    def score(self, user_idx, *args, **kwargs):
         """Scoring using the best searched model"""
-        return self.best_model.score(user_idx, item_idx)
+        return self.best_model.score(user_idx, *args, **kwargs)
+
+    def rank(self, user_idx, item_indices=None, k=-1, **kwargs):
+        """Ranking using the best searched model"""
+        return self.best_model.rank(user_idx, item_indices, k, **kwargs)
 
 
 class GridSearch(BaseSearch):
@@ -263,7 +282,7 @@ def _build_param_set(self):
         """Generate searching points"""
         param_set = []
         keys = [d.name for d in self.space]
-        rng = get_rng(self.model.seed)
+        rng = get_rng(getattr(self.model, "seed", None))
         while len(param_set) < self.n_trails:
             params = [d._sample(rng) for d in self.space]
             param_set.append(dict(zip(keys, params)))

docs/source/user/iamadeveloper.rst

@@ -111,6 +111,10 @@ this, we can use the `cornac.hyperopt` module to perform the searches.
 As shown in the above code, we have defined two methods for hyper-parameter search,
 ``GridSearch`` and ``RandomSearch``.
 
+The same search classes support next-item recommenders when paired with
+``NextItemEvaluation``. In both cases, the evaluation method must include a
+validation split, which is used to select the best parameter settings.
+
 +------------------------------------------+---------------------------------------------+
 | Grid Search                              | Random Search                               |
 +==========================================+=============================================+
@@ -719,4 +723,4 @@ Cornac.
 
   No matter who you are, you could also consider contributing to Cornac,
   with our contributors guide.
-  View :doc:`/developer/index`.
+  View :doc:`/developer/index`.

tests/cornac/test_hyperopt.py

@@ -19,9 +19,9 @@
 import numpy.testing as npt
 
 from cornac.data import Reader
-from cornac.models import MF, BPR
-from cornac.metrics import RMSE, AUC
-from cornac.eval_methods import RatioSplit
+from cornac.models import MF, BPR, SPop
+from cornac.metrics import RMSE, AUC, HitRatio
+from cornac.eval_methods import RatioSplit, NextItemEvaluation
 from cornac.hyperopt import Discrete, Continuous
 from cornac.hyperopt import GridSearch, RandomSearch
 from cornac import Experiment
@@ -70,6 +70,69 @@ def test_random_search(self):
             user_based=False,
         ).run()
 
+    def test_random_search_next_item_recommender(self):
+        data = Reader().read("./tests/sequence.txt", fmt="USIT", sep=" ")
+        eval_method = NextItemEvaluation.from_splits(
+            train_data=data[:35],
+            val_data=data[35:50],
+            test_data=data[50:],
+            fmt="USIT",
+            exclude_unknowns=False,
+            mode="next",
+        )
+        metric = HitRatio(k=5)
+        spop = SPop()
+        spop.seed = 123  # for reproducible RandomSearch sampling
+        rs_spop = RandomSearch(
+            model=spop,
+            space=[Discrete("use_session_popularity", [False, True])],
+            metric=metric,
+            eval_method=eval_method,
+            n_trails=2,
+        )
+
+        test_result, _ = eval_method.evaluate(
+            model=rs_spop,
+            metrics=[metric],
+            user_based=False,
+            show_validation=False,
+        )
+
+        self.assertIsNotNone(rs_spop.best_model)
+        self.assertEqual(rs_spop.best_params, {"use_session_popularity": False})
+        self.assertAlmostEqual(rs_spop.best_score, 11 / 12)
+        self.assertTrue(np.isfinite(test_result.metric_avg_results["HitRatio@5"]))
+
+    def test_grid_search_next_item_recommender(self):
+        data = Reader().read("./tests/sequence.txt", fmt="USIT", sep=" ")
+        eval_method = NextItemEvaluation.from_splits(
+            train_data=data[:35],
+            val_data=data[35:50],
+            test_data=data[50:],
+            fmt="USIT",
+            exclude_unknowns=False,
+            mode="next",
+        )
+        metric = HitRatio(k=5)
+        gs_spop = GridSearch(
+            model=SPop(),
+            space=[Discrete("use_session_popularity", [False, True])],
+            metric=metric,
+            eval_method=eval_method,
+        )
+
+        test_result, _ = eval_method.evaluate(
+            model=gs_spop,
+            metrics=[metric],
+            user_based=False,
+            show_validation=False,
+        )
+
+        self.assertIsNotNone(gs_spop.best_model)
+        self.assertEqual(gs_spop.best_params, {"use_session_popularity": False})
+        self.assertAlmostEqual(gs_spop.best_score, 11 / 12)
+        self.assertTrue(np.isfinite(test_result.metric_avg_results["HitRatio@5"]))
+
 
 if __name__ == "__main__":
     unittest.main()