Merge remote-tracking branch 'origin/main' into feature/py313

Author: chezou

CHANGELOG.md

@@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Python 3.13 support ([#227](https://github.com/MobileTeleSystems/RecTools/pull/227))
 
+## [0.13.0] - 10.04.2025
+
+### Added 
+- `TransformerNegativeSamplerBase` and `CatalogUniformSampler` classes, `negative_sampler_type` and `negative_sampler_kwargs` parameters to transformer-based models ([#275](https://github.com/MobileTeleSystems/RecTools/pull/275))
+- `SimilarityModuleBase`, `DistanceSimilarityModule`, similarity module to `TransformerTorchBackbone` parameters to transformer-based models `similarity_module_type`, `similarity_module_kwargs` ([#272](https://github.com/MobileTeleSystems/RecTools/pull/272))
+- `out_dim` property to `IdEmbeddingsItemNet`, `CatFeaturesItemNet` and `SumOfEmbeddingsConstructor` ([#276](https://github.com/MobileTeleSystems/RecTools/pull/276))
+- `TransformerBackboneBase`, `backbone_type` and `backbone_kwargs` parameters to transformer-based models ([#277](https://github.com/MobileTeleSystems/RecTools/pull/277))
+- `sampled_softmax` loss option for transformer models ([#274](https://github.com/MobileTeleSystems/RecTools/pull/274))
+
+
 ## [0.12.0] - 24.02.2025
 
 ### Added

examples/tutorials/transformers_advanced_training_guide.ipynb

@@ -412,15 +412,15 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch,step,train_loss,val_loss\r",
+      "epoch,step,train_loss,val_loss\r\n",
       "\r\n",
-      "0,1,,22.365339279174805\r",
+      "0,1,,22.365339279174805\r\n",
       "\r\n",
-      "0,1,22.38391876220703,\r",
+      "0,1,22.38391876220703,\r\n",
       "\r\n",
-      "1,3,,22.189851760864258\r",
+      "1,3,,22.189851760864258\r\n",
       "\r\n",
-      "1,3,22.898216247558594,\r",
+      "1,3,22.898216247558594,\r\n",
       "\r\n"
      ]
     }
@@ -526,23 +526,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch,step,train_loss,val_loss\r",
+      "epoch,step,train_loss,val_loss\r\n",
       "\r\n",
-      "0,1,,22.343637466430664\r",
+      "0,1,,22.343637466430664\r\n",
       "\r\n",
-      "0,1,22.36273765563965,\r",
+      "0,1,22.36273765563965,\r\n",
       "\r\n",
-      "1,3,,22.159835815429688\r",
+      "1,3,,22.159835815429688\r\n",
       "\r\n",
-      "1,3,22.33755874633789,\r",
+      "1,3,22.33755874633789,\r\n",
       "\r\n",
-      "2,5,,21.94308853149414\r",
+      "2,5,,21.94308853149414\r\n",
       "\r\n",
-      "2,5,22.244243621826172,\r",
+      "2,5,22.244243621826172,\r\n",
       "\r\n",
-      "3,7,,21.702259063720703\r",
+      "3,7,,21.702259063720703\r\n",
       "\r\n",
-      "3,7,22.196012496948242,\r",
+      "3,7,22.196012496948242,\r\n",
       "\r\n"
      ]
     }
@@ -898,7 +898,7 @@
     "    ) -> None:\n",
     "        logits = outputs[\"logits\"]\n",
     "        if logits is None:\n",
-    "            logits = pl_module.torch_model.encode_sessions(batch[\"x\"], pl_module.item_embs)[:, -1, :]\n",
+    "            logits = pl_module.torch_model.encode_sessions(batch, pl_module.item_embs)[:, -1, :]\n",
     "        _, sorted_batch_recos = logits.topk(k=self.top_k)\n",
     "\n",
     "        batch_recos = sorted_batch_recos.tolist()\n",
@@ -2039,9 +2039,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "rectools",
+   "display_name": ".venv",
    "language": "python",
-   "name": "rectools"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -2053,7 +2053,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.12"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

examples/tutorials/transformers_tutorial.ipynb

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "RecTools"
-version = "0.12.0"
+version = "0.13.0"
 description = "An easy-to-use Python library for building recommendation systems"
 license = "Apache-2.0"
 authors = [

pyproject.toml

@@ -46,6 +46,11 @@ def get_all_embeddings(self) -> torch.Tensor:
         """Return item embeddings."""
         raise NotImplementedError()
 
+    @property
+    def out_dim(self) -> int:
+        """Return item embedding output dimension."""
+        raise NotImplementedError()
+
     @property
     def device(self) -> torch.device:
         """Return ItemNet device."""
@@ -222,6 +227,11 @@ def from_dataset_schema(
             )
         return None
 
+    @property
+    def out_dim(self) -> int:
+        """Return categorical item embedding output dimension."""
+        return self.embedding_bag.embedding_dim
+
 
 class IdEmbeddingsItemNet(ItemNetBase):
     """
@@ -317,6 +327,11 @@ def from_dataset_schema(
         n_items = dataset_schema.items.n_hot
         return cls(n_factors, n_items, dropout_rate)
 
+    @property
+    def out_dim(self) -> int:
+        """Return item embedding output dimension."""
+        return self.ids_emb.embedding_dim
+
 
 class ItemNetConstructorBase(ItemNetBase):
     """
@@ -467,3 +482,8 @@ def forward(self, items: torch.Tensor) -> torch.Tensor:
             item_emb = self.item_net_blocks[idx_block](items)
             item_embs.append(item_emb)
         return torch.sum(torch.stack(item_embs, dim=0), dim=0)
+
+    @property
+    def out_dim(self) -> int:
+        """Return item net constructor output dimension."""
+        return self.item_net_blocks[0].out_dim

rectools/models/nn/item_net.py

@@ -40,13 +40,15 @@
 )
 from .data_preparator import TransformerDataPreparatorBase
 from .lightning import TransformerLightningModule, TransformerLightningModuleBase
+from .negative_sampler import CatalogUniformSampler, TransformerNegativeSamplerBase
 from .net_blocks import (
     LearnableInversePositionalEncoding,
     PositionalEncodingBase,
     PreLNTransformerLayers,
     TransformerLayersBase,
 )
-from .torch_backbone import TransformerTorchBackbone
+from .similarity import DistanceSimilarityModule, SimilarityModuleBase
+from .torch_backbone import TransformerBackboneBase, TransformerTorchBackbone
 
 InitKwargs = tp.Dict[str, tp.Any]
 
@@ -97,6 +99,26 @@ def _serialize_type_sequence(obj: tp.Sequence[tp.Type]) -> tp.Tuple[str, ...]:
     ),
 ]
 
+SimilarityModuleType = tpe.Annotated[
+    tp.Type[SimilarityModuleBase],
+    BeforeValidator(_get_class_obj),
+    PlainSerializer(
+        func=get_class_or_function_full_path,
+        return_type=str,
+        when_used="json",
+    ),
+]
+
+TransformerBackboneType = tpe.Annotated[
+    tp.Type[TransformerBackboneBase],
+    BeforeValidator(_get_class_obj),
+    PlainSerializer(
+        func=get_class_or_function_full_path,
+        return_type=str,
+        when_used="json",
+    ),
+]
+
 TransformerDataPreparatorType = tpe.Annotated[
     tp.Type[TransformerDataPreparatorBase],
     BeforeValidator(_get_class_obj),
@@ -107,6 +129,16 @@ def _serialize_type_sequence(obj: tp.Sequence[tp.Type]) -> tp.Tuple[str, ...]:
     ),
 ]
 
+TransformerNegativeSamplerType = tpe.Annotated[
+    tp.Type[TransformerNegativeSamplerBase],
+    BeforeValidator(_get_class_obj),
+    PlainSerializer(
+        func=get_class_or_function_full_path,
+        return_type=str,
+        when_used="json",
+    ),
+]
+
 
 ItemNetConstructorType = tpe.Annotated[
     tp.Type[ItemNetConstructorBase],
@@ -183,13 +215,19 @@ class TransformerModelConfig(ModelConfig):
     pos_encoding_type: PositionalEncodingType = LearnableInversePositionalEncoding
     transformer_layers_type: TransformerLayersType = PreLNTransformerLayers
     lightning_module_type: TransformerLightningModuleType = TransformerLightningModule
+    negative_sampler_type: TransformerNegativeSamplerType = CatalogUniformSampler
+    similarity_module_type: SimilarityModuleType = DistanceSimilarityModule
+    backbone_type: TransformerBackboneType = TransformerTorchBackbone
     get_val_mask_func: tp.Optional[ValMaskCallableSerialized] = None
     get_trainer_func: tp.Optional[TrainerCallableSerialized] = None
     data_preparator_kwargs: tp.Optional[InitKwargs] = None
     transformer_layers_kwargs: tp.Optional[InitKwargs] = None
     item_net_constructor_kwargs: tp.Optional[InitKwargs] = None
     pos_encoding_kwargs: tp.Optional[InitKwargs] = None
     lightning_module_kwargs: tp.Optional[InitKwargs] = None
+    negative_sampler_kwargs: tp.Optional[InitKwargs] = None
+    similarity_module_kwargs: tp.Optional[InitKwargs] = None
+    backbone_kwargs: tp.Optional[InitKwargs] = None
 
 
 TransformerModelConfig_T = tp.TypeVar("TransformerModelConfig_T", bound=TransformerModelConfig)
@@ -237,13 +275,19 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-locals
         item_net_constructor_type: tp.Type[ItemNetConstructorBase] = SumOfEmbeddingsConstructor,
         pos_encoding_type: tp.Type[PositionalEncodingBase] = LearnableInversePositionalEncoding,
         lightning_module_type: tp.Type[TransformerLightningModuleBase] = TransformerLightningModule,
+        negative_sampler_type: tp.Type[TransformerNegativeSamplerBase] = CatalogUniformSampler,
+        similarity_module_type: tp.Type[SimilarityModuleBase] = DistanceSimilarityModule,
+        backbone_type: tp.Type[TransformerBackboneBase] = TransformerTorchBackbone,
         get_val_mask_func: tp.Optional[ValMaskCallable] = None,
         get_trainer_func: tp.Optional[TrainerCallable] = None,
         data_preparator_kwargs: tp.Optional[InitKwargs] = None,
         transformer_layers_kwargs: tp.Optional[InitKwargs] = None,
         item_net_constructor_kwargs: tp.Optional[InitKwargs] = None,
         pos_encoding_kwargs: tp.Optional[InitKwargs] = None,
         lightning_module_kwargs: tp.Optional[InitKwargs] = None,
+        negative_sampler_kwargs: tp.Optional[InitKwargs] = None,
+        similarity_module_kwargs: tp.Optional[InitKwargs] = None,
+        backbone_kwargs: tp.Optional[InitKwargs] = None,
         **kwargs: tp.Any,
     ) -> None:
         super().__init__(verbose=verbose)
@@ -268,17 +312,23 @@ def __init__(  # pylint: disable=too-many-arguments, too-many-locals
         self.recommend_batch_size = recommend_batch_size
         self.recommend_torch_device = recommend_torch_device
         self.train_min_user_interactions = train_min_user_interactions
+        self.similarity_module_type = similarity_module_type
         self.item_net_block_types = item_net_block_types
         self.item_net_constructor_type = item_net_constructor_type
         self.pos_encoding_type = pos_encoding_type
         self.lightning_module_type = lightning_module_type
+        self.negative_sampler_type = negative_sampler_type
+        self.backbone_type = backbone_type
         self.get_val_mask_func = get_val_mask_func
         self.get_trainer_func = get_trainer_func
         self.data_preparator_kwargs = data_preparator_kwargs
         self.transformer_layers_kwargs = transformer_layers_kwargs
         self.item_net_constructor_kwargs = item_net_constructor_kwargs
         self.pos_encoding_kwargs = pos_encoding_kwargs
         self.lightning_module_kwargs = lightning_module_kwargs
+        self.negative_sampler_kwargs = negative_sampler_kwargs
+        self.similarity_module_kwargs = similarity_module_kwargs
+        self.backbone_kwargs = backbone_kwargs
 
         self._init_data_preparator()
         self._init_trainer()
@@ -295,12 +345,14 @@ def _get_kwargs(actual_kwargs: tp.Optional[InitKwargs]) -> InitKwargs:
         return kwargs
 
     def _init_data_preparator(self) -> None:
+        requires_negatives = self.lightning_module_type.requires_negatives(self.loss)
         self.data_preparator = self.data_preparator_type(
             session_max_len=self.session_max_len,
             batch_size=self.batch_size,
             dataloader_num_workers=self.dataloader_num_workers,
             train_min_user_interactions=self.train_min_user_interactions,
-            n_negatives=self.n_negatives if self.loss != "softmax" else None,
+            negative_sampler=self._init_negative_sampler() if requires_negatives else None,
+            n_negatives=self.n_negatives if requires_negatives else None,
             get_val_mask_func=self.get_val_mask_func,
             shuffle_train=True,
             **self._get_kwargs(self.data_preparator_kwargs),
@@ -321,6 +373,12 @@ def _init_trainer(self) -> None:
         else:
             self._trainer = self.get_trainer_func()
 
+    def _init_negative_sampler(self) -> TransformerNegativeSamplerBase:
+        return self.negative_sampler_type(
+            n_negatives=self.n_negatives,
+            **self._get_kwargs(self.negative_sampler_kwargs),
+        )
+
     def _construct_item_net(self, dataset: Dataset) -> ItemNetBase:
         return self.item_net_constructor_type.from_dataset(
             dataset,
@@ -356,22 +414,28 @@ def _init_transformer_layers(self) -> TransformerLayersBase:
             **self._get_kwargs(self.transformer_layers_kwargs),
         )
 
-    def _init_torch_model(self, item_model: ItemNetBase) -> TransformerTorchBackbone:
+    def _init_similarity_module(self) -> SimilarityModuleBase:
+        return self.similarity_module_type(**self._get_kwargs(self.similarity_module_kwargs))
+
+    def _init_torch_model(self, item_model: ItemNetBase) -> TransformerBackboneBase:
         pos_encoding_layer = self._init_pos_encoding_layer()
         transformer_layers = self._init_transformer_layers()
-        return TransformerTorchBackbone(
+        similarity_module = self._init_similarity_module()
+        return self.backbone_type(
             n_heads=self.n_heads,
             dropout_rate=self.dropout_rate,
             item_model=item_model,
             pos_encoding_layer=pos_encoding_layer,
             transformer_layers=transformer_layers,
+            similarity_module=similarity_module,
             use_causal_attn=self.use_causal_attn,
             use_key_padding_mask=self.use_key_padding_mask,
+            **self._get_kwargs(self.backbone_kwargs),
         )
 
     def _init_lightning_model(
         self,
-        torch_model: TransformerTorchBackbone,
+        torch_model: TransformerBackboneBase,
         dataset_schema: DatasetSchemaDict,
         item_external_ids: ExternalIds,
         model_config: tp.Dict[str, tp.Any],
@@ -467,7 +531,7 @@ def _recommend_i2i(
         )
 
     @property
-    def torch_model(self) -> TransformerTorchBackbone:
+    def torch_model(self) -> TransformerBackboneBase:
         """Pytorch model."""
         return self.lightning_model.torch_model