From cc0845469548de174f9248499b2756125030f114 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Sun, 22 Dec 2024 01:14:31 +0700 Subject: [PATCH 01/15] bert4rec original, timeline split, ce loss all rank --- configs/train/bert4rec_train_config.json | 10 +++++----- modeling/models/bert4rec.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/train/bert4rec_train_config.json b/configs/train/bert4rec_train_config.json index 787c27c3..91418aa1 100644 --- a/configs/train/bert4rec_train_config.json +++ b/configs/train/bert4rec_train_config.json @@ -1,9 +1,9 @@ { - "experiment_name": "bert4rec_beauty", + "experiment_name": "bert4rec_beauty_dataset_bert_ce_loss", "best_metric": "eval/ndcg@20", "dataset": { - "type": "scientific", - "path_to_data_dir": "../data", + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -43,7 +43,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.2, + "dropout": 0.3, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -52,7 +52,7 @@ "type": "basic", "optimizer": { "type": "adam", - "lr": 1e-4 + "lr": 0.001 }, "clip_grad_threshold": 5.0 }, diff --git a/modeling/models/bert4rec.py b/modeling/models/bert4rec.py index 40f1d331..baf4f774 100644 --- a/modeling/models/bert4rec.py +++ b/modeling/models/bert4rec.py @@ -72,11 +72,11 @@ def forward(self, inputs): ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) - embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) embeddings = torch.einsum( 'bsd,nd->bsn', embeddings, self._item_embeddings.weight ) # (batch_size, seq_len, num_items) - embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) if self.training: # training mode all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) From 949453725eb853d39adb1bdb57185eb808904b9d Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Fri, 3 Jan 2025 16:28:07 +0700 Subject: [PATCH 02/15] implement bert4rec all rank and popular (in batch) --- .../train/bert4rec_train_config_all_rank.json | 146 ++++++++++++++++++ .../train/bert4rec_train_config_popular.json | 146 ++++++++++++++++++ .../samplers/masked_item_prediction.py | 5 +- modeling/models/__init__.py | 2 + modeling/models/bert4rec_all_rank.py | 100 ++++++++++++ modeling/models/bert4rec_popular.py | 120 ++++++++++++++ 6 files changed, 518 insertions(+), 1 deletion(-) create mode 100644 configs/train/bert4rec_train_config_all_rank.json create mode 100644 configs/train/bert4rec_train_config_popular.json create mode 100644 modeling/models/bert4rec_all_rank.py create mode 100644 modeling/models/bert4rec_popular.py diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json new file mode 100644 index 00000000..d0956bd4 --- /dev/null +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -0,0 +1,146 @@ +{ + "experiment_name": "bert4rec_beauty_dataset_bert_ce_loss_all_rank", + "best_metric": "eval/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.7, + "num_negatives_val": 100, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_all_rank", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "ce", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json new file mode 100644 index 00000000..681d90d1 --- /dev/null +++ b/configs/train/bert4rec_train_config_popular.json @@ -0,0 +1,146 @@ +{ + "experiment_name": "bert4rec_beauty_dataset_bert_ce_loss_popular", + "best_metric": "eval/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.7, + "num_negatives_val": 100, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_popular", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "ce", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + } + } + } + ] + } +} diff --git a/modeling/dataset/samplers/masked_item_prediction.py b/modeling/dataset/samplers/masked_item_prediction.py index 0100f731..811c500c 100644 --- a/modeling/dataset/samplers/masked_item_prediction.py +++ b/modeling/dataset/samplers/masked_item_prediction.py @@ -61,7 +61,10 @@ def __getitem__(self, index): 'item.length': len(masked_sequence), 'labels.ids': labels, - 'labels.length': len(labels) + 'labels.length': len(labels), + + 'not_masked_item.ids': item_sequence, + 'not_masked_item.length': len(item_sequence) } diff --git a/modeling/models/__init__.py b/modeling/models/__init__.py index afed8466..9c134136 100644 --- a/modeling/models/__init__.py +++ b/modeling/models/__init__.py @@ -1,6 +1,8 @@ from .base import BaseModel, TorchModel from .bert4rec import Bert4RecModel from .bert4rec_cls import Bert4RecModelCLS +from .bert4rec_all_rank import Bert4RecModelAllRank +from .bert4rec_popular import Bert4RecModelPopular from .cl4srec import Cl4SRecModel from .duorec import DuoRecModel from .graph_seq_rec import GraphSeqRecModel diff --git a/modeling/models/bert4rec_all_rank.py b/modeling/models/bert4rec_all_rank.py new file mode 100644 index 00000000..d9404953 --- /dev/null +++ b/modeling/models/bert4rec_all_rank.py @@ -0,0 +1,100 @@ +from models.base import SequentialTorchModel + +import torch +import torch.nn as nn + + +class Bert4RecModelAllRank(SequentialTorchModel, config_name='bert4rec_all_rank'): + + def __init__( + self, + sequence_prefix, + labels_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='gelu', + layer_norm_eps=1e-5, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=False + ) + self._sequence_prefix = sequence_prefix + self._labels_prefix = labels_prefix + + self._output_projection = nn.Linear( + in_features=embedding_dim, + out_features=embedding_dim + ) + + self._bias = nn.Parameter( + data=torch.zeros(num_items + 2), + requires_grad=True + ) + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + labels_prefix=config['labels_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + embeddings = torch.einsum( + 'bsd,nd->bsn', embeddings, self._item_embeddings.weight + ) # (batch_size, seq_len, num_items) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + + if self.training: # training mode + all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) + embeddings = embeddings[mask] # (all_batch_events, num_items) + labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + + needed_logits = embeddings[labels_mask] # (non_zero_events, num_items) + needed_labels = all_sample_labels[labels_mask] # (non_zero_events) + + return {'logits': needed_logits, 'labels.ids': needed_labels} + else: # eval mode + candidate_scores = self._get_last_embedding(embeddings, mask) # (batch_size, num_items) + candidate_scores[:, 0] = -torch.inf + candidate_scores[:, self._num_items + 1:] = -torch.inf + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices diff --git a/modeling/models/bert4rec_popular.py b/modeling/models/bert4rec_popular.py new file mode 100644 index 00000000..3c3b1b86 --- /dev/null +++ b/modeling/models/bert4rec_popular.py @@ -0,0 +1,120 @@ +from models.base import SequentialTorchModel + +import torch +import torch.nn as nn + + +class Bert4RecModelPopular(SequentialTorchModel, config_name='bert4rec_popular'): + + def __init__( + self, + sequence_prefix, + labels_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='gelu', + layer_norm_eps=1e-5, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=False + ) + self._sequence_prefix = sequence_prefix + self._labels_prefix = labels_prefix + + self._output_projection = nn.Linear( + in_features=embedding_dim, + out_features=embedding_dim + ) + + self._bias = nn.Parameter( + data=torch.zeros(num_items + 2), + requires_grad=True + ) + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + labels_prefix=config['labels_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + + if self.training: # training mode + # TODO: move 'not_masked_item' to config + all_sample_not_masked = inputs['not_masked_item.ids'] # (all_batch_events) + + random_indices = torch.randperm(all_sample_not_masked.shape[0])[:embeddings.shape[0]] # (batch_size) + random_in_batch_negative_ids = all_sample_not_masked[random_indices] # (batch_size) + random_in_batch_negative_embeddings = self._item_embeddings.weight[random_in_batch_negative_ids] # (batch_size) + + embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) + labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + non_zero_embeddings = embeddings[labels_mask] # (non_zero_events, embedding_dim) + non_zero_labels = all_sample_labels[labels_mask] # (non_zero_events) + + # non_zero_samples_logits = torch.einsum( + # 'bd,nd->bn', non_zero_embeddings, random_in_batch_negative_embeddings + # ) # (non_zero_events, num_negatives=batch_size) + non_zero_samples_logits = non_zero_embeddings @ random_in_batch_negative_embeddings.T # (non_zero_events, num_negatives=batch_size) + non_zero_labels_embeddings = self._item_embeddings.weight[non_zero_labels] # (non_zero_events, embedding_dim) + non_zero_labels_logits = (non_zero_embeddings @ non_zero_labels_embeddings.T).diagonal().unsqueeze(1) # (non_zero_events, 1) + + needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1=batch_size + 1) + # needed_labels = all_sample_labels[labels_mask] # (non_zero_events) + + needed_labels = torch.zeros(len(needed_logits), dtype=torch.long) # (non_zero_events) + + return {'logits': needed_logits, 'labels.ids': needed_labels} + else: + # eval mode + embeddings = torch.einsum( + 'bsd,nd->bsn', embeddings, self._item_embeddings.weight + ) # (batch_size, seq_len, num_items) + + candidate_scores = self._get_last_embedding(embeddings, mask) # (batch_size, num_items) + candidate_scores[:, 0] = -torch.inf + candidate_scores[:, self._num_items + 1:] = -torch.inf + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices From d716a632646eaa7a4c30621d1556387d0543ca70 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Sat, 4 Jan 2025 23:56:45 +0700 Subject: [PATCH 03/15] implement random sampler by popularity for bert4rec --- .../train/bert4rec_train_config_popular.json | 4 +- configs/train/sasrec_train_config.json | 6 +- .../dataset/negative_samplers/__init__.py | 4 +- .../negative_samplers/random_by_popularity.py | 59 +++++++++ .../samplers/masked_item_prediction.py | 49 +++++-- modeling/models/__init__.py | 1 + modeling/models/bert4rec_in_batch.py | 121 ++++++++++++++++++ modeling/models/bert4rec_popular.py | 23 +--- 8 files changed, 233 insertions(+), 34 deletions(-) create mode 100644 modeling/dataset/negative_samplers/random_by_popularity.py create mode 100644 modeling/models/bert4rec_in_batch.py diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json index 681d90d1..1896f0b8 100644 --- a/configs/train/bert4rec_train_config_popular.json +++ b/configs/train/bert4rec_train_config_popular.json @@ -8,9 +8,9 @@ "max_sequence_length": 50, "samplers": { "mask_prob": 0.7, - "num_negatives_val": 100, + "num_negatives_train": 100, "type": "masked_item_prediction", - "negative_sampler_type": "random" + "negative_sampler_type": "random_by_popularity" } }, "dataloader": { diff --git a/configs/train/sasrec_train_config.json b/configs/train/sasrec_train_config.json index 1336b91c..08302377 100644 --- a/configs/train/sasrec_train_config.json +++ b/configs/train/sasrec_train_config.json @@ -3,13 +3,13 @@ "best_metric": "eval/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { - "num_negatives_val": 100, + "num_negatives_train": 100, "type": "next_item_prediction", - "negative_sampler_type": "random" + "negative_sampler_type": "popular" } }, "dataloader": { diff --git a/modeling/dataset/negative_samplers/__init__.py b/modeling/dataset/negative_samplers/__init__.py index 498de21f..70db4559 100644 --- a/modeling/dataset/negative_samplers/__init__.py +++ b/modeling/dataset/negative_samplers/__init__.py @@ -1,9 +1,11 @@ from .base import BaseNegativeSampler from .popular import PopularNegativeSampler from .random import RandomNegativeSampler +from .random_by_popularity import RandomByPopularityNegativeSampler __all__ = [ 'BaseNegativeSampler', 'PopularNegativeSampler', - 'RandomNegativeSampler' + 'RandomNegativeSampler', + 'RandomByPopularityNegativeSampler' ] diff --git a/modeling/dataset/negative_samplers/random_by_popularity.py b/modeling/dataset/negative_samplers/random_by_popularity.py new file mode 100644 index 00000000..4d209dfa --- /dev/null +++ b/modeling/dataset/negative_samplers/random_by_popularity.py @@ -0,0 +1,59 @@ +from dataset.negative_samplers.base import BaseNegativeSampler + +from collections import Counter +import torch + + +class RandomByPopularityNegativeSampler(BaseNegativeSampler, config_name='random_by_popularity'): + + def __init__( + self, + dataset, + num_users, + num_items + ): + super().__init__( + dataset=dataset, + num_users=num_users, + num_items=num_items + ) + + self._item_popularity = self._compute_item_popularity() + + @classmethod + def create_from_config(cls, _, **kwargs): + return cls( + dataset=kwargs['dataset'], + num_users=kwargs['num_users'], + num_items=kwargs['num_items'] + ) + + def _compute_item_popularity(self): + popularity = Counter() + + for sample in self._dataset: + for item_id in sample['item.ids']: + popularity[item_id] += 1 + + # Convert to tensor for efficient sampling + items = list(popularity.keys()) + weights = torch.tensor(list(popularity.values()), dtype=torch.float32) + return {"items": items, "weights": weights} + + def generate_negative_samples(self, sample, num_negatives): + user_id = sample['user.ids'][0] + seen_items = set(self._seen_items[user_id]) # Convert to set for faster lookup + + items = self._item_popularity["items"] + weights = self._item_popularity["weights"] + + negatives = [] + while len(negatives) < num_negatives: + sampled_indices = torch.multinomial(weights, num_samples=num_negatives, replacement=False) + sampled_items = [items[idx] for idx in sampled_indices] + + for item in sampled_items: + if item not in seen_items and len(negatives) < num_negatives: + negatives.append(item) + + return negatives diff --git a/modeling/dataset/samplers/masked_item_prediction.py b/modeling/dataset/samplers/masked_item_prediction.py index 811c500c..f9170f19 100644 --- a/modeling/dataset/samplers/masked_item_prediction.py +++ b/modeling/dataset/samplers/masked_item_prediction.py @@ -1,4 +1,5 @@ from dataset.samplers.base import TrainSampler, EvalSampler +from dataset.negative_samplers.base import BaseNegativeSampler import copy import numpy as np @@ -6,21 +7,27 @@ class MaskedItemPredictionTrainSampler(TrainSampler, config_name='masked_item_prediction'): - def __init__(self, dataset, num_users, num_items, mask_prob=0.0): + def __init__(self, dataset, num_users, num_items, negative_sampler, mask_prob=0.0, num_negatives=0): super().__init__() self._dataset = dataset self._num_users = num_users self._num_items = num_items self._mask_item_idx = self._num_items + 1 self._mask_prob = mask_prob + self._negative_sampler = negative_sampler + self._num_negatives = num_negatives @classmethod def create_from_config(cls, config, **kwargs): + negative_sampler = BaseNegativeSampler.create_from_config({'type': config['negative_sampler_type']}, **kwargs) + return cls( dataset=kwargs['dataset'], num_users=kwargs['num_users'], num_items=kwargs['num_items'], - mask_prob=config.get('mask_prob', 0.0) + negative_sampler=negative_sampler, + mask_prob=config.get('mask_prob', 0.0), + num_negatives=config.get('num_negatives_train', 0) ) def __getitem__(self, index): @@ -53,20 +60,38 @@ def __getitem__(self, index): masked_sequence[-1] = self._mask_item_idx labels[-1] = item_sequence[-1] - return { - 'user.ids': sample['user.ids'], - 'user.length': sample['user.length'], + if self._num_negatives == 0: + return { + 'user.ids': sample['user.ids'], + 'user.length': sample['user.length'], - 'item.ids': masked_sequence, - 'item.length': len(masked_sequence), + 'item.ids': masked_sequence, + 'item.length': len(masked_sequence), - 'labels.ids': labels, - 'labels.length': len(labels), + 'labels.ids': labels, + 'labels.length': len(labels), - 'not_masked_item.ids': item_sequence, - 'not_masked_item.length': len(item_sequence) - } + 'not_masked_item.ids': item_sequence, + 'not_masked_item.length': len(item_sequence) + } + else: + negative_sequence = self._negative_sampler.generate_negative_samples( + sample, self._num_negatives + ) + + return { + 'user.ids': sample['user.ids'], + 'user.length': sample['user.length'], + + 'item.ids': masked_sequence, + 'item.length': len(masked_sequence), + + 'labels.ids': labels, + 'labels.length': len(labels), + 'negative_item.ids': negative_sequence, + 'negative_item.length': len(negative_sequence) + } class MaskedItemPredictionEvalSampler(EvalSampler, config_name='masked_item_prediction'): diff --git a/modeling/models/__init__.py b/modeling/models/__init__.py index 9c134136..46812f59 100644 --- a/modeling/models/__init__.py +++ b/modeling/models/__init__.py @@ -2,6 +2,7 @@ from .bert4rec import Bert4RecModel from .bert4rec_cls import Bert4RecModelCLS from .bert4rec_all_rank import Bert4RecModelAllRank +from .bert4rec_in_batch import Bert4RecModelInBatch from .bert4rec_popular import Bert4RecModelPopular from .cl4srec import Cl4SRecModel from .duorec import DuoRecModel diff --git a/modeling/models/bert4rec_in_batch.py b/modeling/models/bert4rec_in_batch.py new file mode 100644 index 00000000..e5cddb48 --- /dev/null +++ b/modeling/models/bert4rec_in_batch.py @@ -0,0 +1,121 @@ +from models.base import SequentialTorchModel + +import torch +import torch.nn as nn + + +class Bert4RecModelInBatch(SequentialTorchModel, config_name='bert4rec_in_batch'): + + def __init__( + self, + sequence_prefix, + labels_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='gelu', + layer_norm_eps=1e-5, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=False + ) + self._sequence_prefix = sequence_prefix + self._labels_prefix = labels_prefix + + self._output_projection = nn.Linear( + in_features=embedding_dim, + out_features=embedding_dim + ) + + self._bias = nn.Parameter( + data=torch.zeros(num_items + 2), + requires_grad=True + ) + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + labels_prefix=config['labels_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + + if self.training: # training mode + # TODO: move 'not_masked_item' to config + all_sample_not_masked = inputs['not_masked_item.ids'] # (all_batch_events) + + random_indices = torch.randperm(all_sample_not_masked.shape[0])[:embeddings.shape[0]] # (batch_size) + random_in_batch_negative_ids = all_sample_not_masked[random_indices] # (batch_size) + random_in_batch_negative_embeddings = self._item_embeddings.weight[random_in_batch_negative_ids] # (batch_size) + + embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) + labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + non_zero_embeddings = embeddings[labels_mask] # (non_zero_events, embedding_dim) + non_zero_labels = all_sample_labels[labels_mask] # (non_zero_events) + + # non_zero_samples_logits = torch.einsum( + # 'bd,nd->bn', non_zero_embeddings, random_in_batch_negative_embeddings + # ) # (non_zero_events, num_negatives=batch_size) + non_zero_samples_logits = non_zero_embeddings @ random_in_batch_negative_embeddings.T # (non_zero_events, num_negatives=batch_size) + non_zero_labels_embeddings = self._item_embeddings.weight[non_zero_labels] # (non_zero_events, embedding_dim) + # non_zero_labels_logits = (non_zero_embeddings @ non_zero_labels_embeddings.T).diagonal().unsqueeze(1) # (non_zero_events, 1) + non_zero_labels_logits = (non_zero_embeddings * non_zero_labels_embeddings).sum(dim=-1).unsqueeze(1) # (non_zero_events, 1) + + needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1=batch_size + 1) + # needed_labels = all_sample_labels[labels_mask] # (non_zero_events) + + needed_labels = torch.zeros(len(needed_logits), dtype=torch.long) # (non_zero_events) + + return {'logits': needed_logits, 'labels.ids': needed_labels} + else: + # eval mode + embeddings = torch.einsum( + 'bsd,nd->bsn', embeddings, self._item_embeddings.weight + ) # (batch_size, seq_len, num_items) + + candidate_scores = self._get_last_embedding(embeddings, mask) # (batch_size, num_items) + candidate_scores[:, 0] = -torch.inf + candidate_scores[:, self._num_items + 1:] = -torch.inf + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices diff --git a/modeling/models/bert4rec_popular.py b/modeling/models/bert4rec_popular.py index 3c3b1b86..003fc506 100644 --- a/modeling/models/bert4rec_popular.py +++ b/modeling/models/bert4rec_popular.py @@ -72,16 +72,12 @@ def forward(self, inputs): ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) - # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) - # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) - if self.training: # training mode + if self.training: # training mode # TODO: move 'not_masked_item' to config - all_sample_not_masked = inputs['not_masked_item.ids'] # (all_batch_events) + negative_items = inputs['negative_item.ids'] # (num_negatives) - random_indices = torch.randperm(all_sample_not_masked.shape[0])[:embeddings.shape[0]] # (batch_size) - random_in_batch_negative_ids = all_sample_not_masked[random_indices] # (batch_size) - random_in_batch_negative_embeddings = self._item_embeddings.weight[random_in_batch_negative_ids] # (batch_size) + negative_embeddings = self._item_embeddings.weight[negative_items] # (num_negatives) embeddings = embeddings[mask] # (all_batch_events, embedding_dim) all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) @@ -89,21 +85,16 @@ def forward(self, inputs): non_zero_embeddings = embeddings[labels_mask] # (non_zero_events, embedding_dim) non_zero_labels = all_sample_labels[labels_mask] # (non_zero_events) - # non_zero_samples_logits = torch.einsum( - # 'bd,nd->bn', non_zero_embeddings, random_in_batch_negative_embeddings - # ) # (non_zero_events, num_negatives=batch_size) - non_zero_samples_logits = non_zero_embeddings @ random_in_batch_negative_embeddings.T # (non_zero_events, num_negatives=batch_size) + non_zero_samples_logits = non_zero_embeddings @ negative_embeddings.T # (non_zero_events, num_negatives) non_zero_labels_embeddings = self._item_embeddings.weight[non_zero_labels] # (non_zero_events, embedding_dim) - non_zero_labels_logits = (non_zero_embeddings @ non_zero_labels_embeddings.T).diagonal().unsqueeze(1) # (non_zero_events, 1) + non_zero_labels_logits = (non_zero_embeddings * non_zero_labels_embeddings).sum(dim=-1).unsqueeze(1) # (non_zero_events, 1) - needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1=batch_size + 1) - # needed_labels = all_sample_labels[labels_mask] # (non_zero_events) + needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1) needed_labels = torch.zeros(len(needed_logits), dtype=torch.long) # (non_zero_events) return {'logits': needed_logits, 'labels.ids': needed_labels} - else: - # eval mode + else: # eval mode embeddings = torch.einsum( 'bsd,nd->bsn', embeddings, self._item_embeddings.weight ) # (batch_size, seq_len, num_items) From b2b24d4677973606ad309898854751c642f9c680 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Mon, 6 Jan 2025 00:11:55 +0700 Subject: [PATCH 04/15] implement populat, all rank and in batch for bert4rec and sasrec --- .../train/bert4rec_train_config_all_rank.json | 1 - .../train/bert4rec_train_config_popular.json | 2 +- configs/train/sasrec_train_config.json | 2 +- .../train/sasrec_train_config_all_rank.json | 167 +++++++++++++ .../train/sasrec_train_config_popular.json | 168 ++++++++++++++ modeling/loss/base.py | 4 +- modeling/models/__init__.py | 2 +- modeling/models/bert4rec_popular.py | 10 +- modeling/models/sasrec.py | 219 +++++++++++++++++- 9 files changed, 563 insertions(+), 12 deletions(-) create mode 100644 configs/train/sasrec_train_config_all_rank.json create mode 100644 configs/train/sasrec_train_config_popular.json diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json index d0956bd4..15139334 100644 --- a/configs/train/bert4rec_train_config_all_rank.json +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -8,7 +8,6 @@ "max_sequence_length": 50, "samplers": { "mask_prob": 0.7, - "num_negatives_val": 100, "type": "masked_item_prediction", "negative_sampler_type": "random" } diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json index 1896f0b8..0306f981 100644 --- a/configs/train/bert4rec_train_config_popular.json +++ b/configs/train/bert4rec_train_config_popular.json @@ -1,5 +1,5 @@ { - "experiment_name": "bert4rec_beauty_dataset_bert_ce_loss_popular", + "experiment_name": "bert4rec_popular_beauty", "best_metric": "eval/ndcg@20", "dataset": { "type": "sequence", diff --git a/configs/train/sasrec_train_config.json b/configs/train/sasrec_train_config.json index a264d542..033bc032 100644 --- a/configs/train/sasrec_train_config.json +++ b/configs/train/sasrec_train_config.json @@ -9,7 +9,7 @@ "samplers": { "num_negatives_train": 100, "type": "next_item_prediction", - "negative_sampler_type": "popular" + "negative_sampler_type": "random_by_popularity" } }, "dataloader": { diff --git a/configs/train/sasrec_train_config_all_rank.json b/configs/train/sasrec_train_config_all_rank.json new file mode 100644 index 00000000..5c6383df --- /dev/null +++ b/configs/train/sasrec_train_config_all_rank.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_beauty", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_all_rank", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_popular.json b/configs/train/sasrec_train_config_popular.json new file mode 100644 index 00000000..a5436e2e --- /dev/null +++ b/configs/train/sasrec_train_config_popular.json @@ -0,0 +1,168 @@ +{ + "experiment_name": "sasrec_popular_beauty", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "num_negatives_train": 100, + "type": "next_item_prediction", + "negative_sampler_type": "random_by_popularity" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_popular", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/modeling/loss/base.py b/modeling/loss/base.py index dc03d6cb..dd4481c7 100644 --- a/modeling/loss/base.py +++ b/modeling/loss/base.py @@ -255,8 +255,8 @@ def __init__( self._output_prefix = output_prefix def forward(self, inputs): - positive_scores = inputs[self._positive_prefix] # (x, embedding_dim) - negative_scores = inputs[self._negative_prefix] # (x, embedding_dim) + positive_scores = inputs[self._positive_prefix] # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) assert positive_scores.shape[0] == negative_scores.shape[0] positive_loss = torch.log(nn.functional.sigmoid(positive_scores)).sum(dim=-1) # (x) diff --git a/modeling/models/__init__.py b/modeling/models/__init__.py index 5ce988a9..80d60b44 100644 --- a/modeling/models/__init__.py +++ b/modeling/models/__init__.py @@ -15,6 +15,6 @@ from .pop import PopModel from .pure_mf import PureMF from .random import RandomModel -from .sasrec import SasRecModel, SasRecInBatchModel +from .sasrec import SasRecModel, SasRecModelAllRank, SasRecModelPopular, SasRecInBatchModel from .sasrec_ce import SasRecCeModel from .s3rec import S3RecModel diff --git a/modeling/models/bert4rec_popular.py b/modeling/models/bert4rec_popular.py index 003fc506..c56de255 100644 --- a/modeling/models/bert4rec_popular.py +++ b/modeling/models/bert4rec_popular.py @@ -74,18 +74,22 @@ def forward(self, inputs): embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) if self.training: # training mode - # TODO: move 'not_masked_item' to config + # TODO: move 'negative_item' to config negative_items = inputs['negative_item.ids'] # (num_negatives) - negative_embeddings = self._item_embeddings.weight[negative_items] # (num_negatives) + negative_embeddings = self._item_embeddings.weight[negative_items] # (batch_size * num_negatives, embedding_dim) + negative_embeddings = negative_embeddings.reshape(len(all_sample_lengths), int(negative_embeddings.size(0) / len(all_sample_lengths)), negative_embeddings.size(1)) # (batch_size, num_negatives, embedding_dim) + negative_embeddings = torch.repeat_interleave(negative_embeddings, all_sample_lengths, dim=0) # (all_batch_events, num_negatives, embedding_dim) embeddings = embeddings[mask] # (all_batch_events, embedding_dim) all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + non_zero_negative_embeddings = negative_embeddings[labels_mask] # (non_zero_events, num_negatives, embedding_dim) non_zero_embeddings = embeddings[labels_mask] # (non_zero_events, embedding_dim) non_zero_labels = all_sample_labels[labels_mask] # (non_zero_events) - non_zero_samples_logits = non_zero_embeddings @ negative_embeddings.T # (non_zero_events, num_negatives) + non_zero_samples_logits = torch.einsum("bd,bnd->bn", non_zero_embeddings, non_zero_negative_embeddings) # (non_zero_events, num_negatives) + # non_zero_samples_logits = non_zero_embeddings @ non_zero_negative_embeddings.T # (non_zero_events, num_negatives) non_zero_labels_embeddings = self._item_embeddings.weight[non_zero_labels] # (non_zero_events, embedding_dim) non_zero_labels_logits = (non_zero_embeddings * non_zero_labels_embeddings).sum(dim=-1).unsqueeze(1) # (non_zero_events, 1) diff --git a/modeling/models/sasrec.py b/modeling/models/sasrec.py index 1ef5c9e7..869ef653 100644 --- a/modeling/models/sasrec.py +++ b/modeling/models/sasrec.py @@ -119,6 +119,218 @@ def forward(self, inputs): return indices +class SasRecModelAllRank(SequentialTorchModel, config_name='sasrec_all_rank'): + + def __init__( + self, + sequence_prefix, + positive_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='relu', + layer_norm_eps=1e-9, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=True + ) + self._sequence_prefix = sequence_prefix + self._positive_prefix = positive_prefix + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + positive_prefix=config['positive_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + if self.training: # training mode + all_positive_sample_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) + + all_sample_embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + + all_embeddings = self._item_embeddings.weight # (num_items + 2, embedding_dim) + + # a -- all_batch_events, n -- num_items + 2, d -- embedding_dim + all_scores = torch.einsum( + 'ad,nd->an', + all_sample_embeddings, + all_embeddings + ) # (all_batch_events, num_items + 2) + + positive_scores = torch.gather( + input=all_scores, + dim=1, + index=all_positive_sample_events[..., None] + ) # (all_batch_items, 1) + + sample_ids, _ = create_masked_tensor( + data=all_sample_events, + lengths=all_sample_lengths + ) # (batch_size, seq_len) + + sample_ids = torch.repeat_interleave(sample_ids, all_sample_lengths, dim=0) # (all_batch_events, seq_len) + + negative_scores = torch.scatter( + input=all_scores, + dim=1, + index=sample_ids, + src=torch.ones_like(sample_ids) * (-torch.inf) + ) # (all_batch_events, num_items + 2) + negative_scores[:, 0] = -torch.inf # Padding idx + negative_scores[:, self._num_items + 1:] = -torch.inf # Mask idx + + return { + 'positive_scores': positive_scores, + 'negative_scores': negative_scores + } + else: # eval mode + last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) + # b - batch_size, n - num_candidates, d - embedding_dim + candidate_scores = torch.einsum( + 'bd,nd->bn', + last_embeddings, + self._item_embeddings.weight + ) # (batch_size, num_items + 2) + candidate_scores[:, 0] = -torch.inf # Padding id + candidate_scores[:, self._num_items + 1:] = -torch.inf # Mask id + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices + + +class SasRecModelPopular(SequentialTorchModel, config_name='sasrec_popular'): + + def __init__( + self, + sequence_prefix, + positive_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='relu', + layer_norm_eps=1e-9, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=True + ) + self._sequence_prefix = sequence_prefix + self._positive_prefix = positive_prefix + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + positive_prefix=config['positive_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + if self.training: # training mode + # TODO: move 'negative' to config + negative_items = inputs['negative.ids'] # (num_negatives) + negative_embeddings = self._item_embeddings.weight[negative_items] # (num_negatives, embedding_dim) + negative_embeddings = negative_embeddings.reshape(len(all_sample_lengths), int(negative_embeddings.size(0) / len(all_sample_lengths)), negative_embeddings.size(1)) # (batch_size, num_negatives, embedding_dim) + negative_embeddings = torch.repeat_interleave(negative_embeddings, all_sample_lengths, dim=0) # (all_batch_events, num_negatives, embedding_dim) + + all_sample_embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + negative_scores = torch.einsum("bd,bnd->bn", all_sample_embeddings, negative_embeddings) # (non_zero_events, num_negatives) + # negative_scores = all_sample_embeddings @ negative_embeddings.T + + all_positive_sample_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) + all_positive_sample_embeddings = self._item_embeddings.weight[ + all_positive_sample_events] # (non_zero_events, embedding_dim) + + positive_scores = (all_sample_embeddings * all_positive_sample_embeddings).sum(dim=-1) + + return { + 'positive_scores': positive_scores, + 'negative_scores': negative_scores + } + else: # eval mode + last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) + # b - batch_size, n - num_candidates, d - embedding_dim + candidate_scores = torch.einsum( + 'bd,nd->bn', + last_embeddings, + self._item_embeddings.weight + ) # (batch_size, num_items + 2) + candidate_scores[:, 0] = -torch.inf # Padding id + candidate_scores[:, self._num_items + 1:] = -torch.inf # Mask id + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices + class SasRecInBatchModel(SasRecModel, config_name='sasrec_in_batch'): @@ -194,10 +406,11 @@ def forward(self, inputs): in_batch_negative_ids ) # (batch_size, embedding_dim) + in_batch_negative_embeddings = in_batch_queries_embeddings @ in_batch_negative_embeddings.T + return { - 'query_embeddings': in_batch_queries_embeddings, - 'positive_embeddings': in_batch_positive_embeddings, - 'negative_embeddings': in_batch_negative_embeddings + 'positive_scores': in_batch_positive_embeddings, + 'negative_scores': in_batch_negative_embeddings } else: # eval mode last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) From ed4098fda44aefd9f0b556a4be11f09df8ade8a8 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Mon, 6 Jan 2025 11:54:26 +0700 Subject: [PATCH 05/15] change configs experiment_names --- configs/train/bert4rec_train_config_all_rank.json | 2 +- configs/train/sasrec_train_config_all_rank.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json index 15139334..3d5cc1f7 100644 --- a/configs/train/bert4rec_train_config_all_rank.json +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -1,5 +1,5 @@ { - "experiment_name": "bert4rec_beauty_dataset_bert_ce_loss_all_rank", + "experiment_name": "bert4rec_all_rank_beauty", "best_metric": "eval/ndcg@20", "dataset": { "type": "sequence", diff --git a/configs/train/sasrec_train_config_all_rank.json b/configs/train/sasrec_train_config_all_rank.json index 5c6383df..7bae9db9 100644 --- a/configs/train/sasrec_train_config_all_rank.json +++ b/configs/train/sasrec_train_config_all_rank.json @@ -1,5 +1,5 @@ { - "experiment_name": "sasrec_beauty", + "experiment_name": "sasrec_all_rank_beauty", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", From 53c76215bf6f58819a4afb95779e025e86d4aaf7 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Mon, 6 Jan 2025 12:18:41 +0700 Subject: [PATCH 06/15] try fix device --- modeling/models/bert4rec_popular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modeling/models/bert4rec_popular.py b/modeling/models/bert4rec_popular.py index c56de255..18c403b9 100644 --- a/modeling/models/bert4rec_popular.py +++ b/modeling/models/bert4rec_popular.py @@ -95,7 +95,7 @@ def forward(self, inputs): needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1) - needed_labels = torch.zeros(len(needed_logits), dtype=torch.long) # (non_zero_events) + needed_labels = torch.zeros(len(needed_logits), dtype=torch.long, device=needed_logits.device) # (non_zero_events) return {'logits': needed_logits, 'labels.ids': needed_labels} else: # eval mode From 6542b3b0e1a0df2b42476950d3db96b290c02faa Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Wed, 8 Jan 2025 22:17:04 +0700 Subject: [PATCH 07/15] fix nans --- configs/train/sasrec_train_config.json | 3 +-- modeling/loss/base.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/configs/train/sasrec_train_config.json b/configs/train/sasrec_train_config.json index 033bc032..ad8dc2c5 100644 --- a/configs/train/sasrec_train_config.json +++ b/configs/train/sasrec_train_config.json @@ -7,9 +7,8 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "num_negatives_train": 100, "type": "next_item_prediction", - "negative_sampler_type": "random_by_popularity" + "negative_sampler_type": "random" } }, "dataloader": { diff --git a/modeling/loss/base.py b/modeling/loss/base.py index dd4481c7..ef8ba88c 100644 --- a/modeling/loss/base.py +++ b/modeling/loss/base.py @@ -259,8 +259,8 @@ def forward(self, inputs): negative_scores = inputs[self._negative_prefix] # (x, num_negatives) assert positive_scores.shape[0] == negative_scores.shape[0] - positive_loss = torch.log(nn.functional.sigmoid(positive_scores)).sum(dim=-1) # (x) - negative_loss = torch.log(1.0 - nn.functional.sigmoid(negative_scores)).sum(dim=-1) # (x) + positive_loss = torch.log(nn.functional.sigmoid(positive_scores) + 1e-9).sum(dim=-1) # (x) + negative_loss = torch.log(1.0 - nn.functional.sigmoid(negative_scores) + 1e-9).sum(dim=-1) # (x) loss = positive_loss + negative_loss # (x) loss = -loss.sum() # (1) From b7b112bb00525110dd2df159a15a0df2adf8526b Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Wed, 8 Jan 2025 22:37:28 +0700 Subject: [PATCH 08/15] add coverage and print metrics --- .../train/bert4rec_train_config_all_rank.json | 12 +++++++++ .../train/bert4rec_train_config_popular.json | 12 +++++++++ modeling/train.py | 27 +++++++++++++++++-- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json index 3d5cc1f7..11097be0 100644 --- a/configs/train/bert4rec_train_config_all_rank.json +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -137,6 +137,18 @@ "recall@20": { "type": "recall", "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 } } } diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json index 0306f981..db7ab506 100644 --- a/configs/train/bert4rec_train_config_popular.json +++ b/configs/train/bert4rec_train_config_popular.json @@ -138,6 +138,18 @@ "recall@20": { "type": "recall", "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 } } } diff --git a/modeling/train.py b/modeling/train.py index d865620d..1090d723 100644 --- a/modeling/train.py +++ b/modeling/train.py @@ -1,12 +1,13 @@ import utils from utils import parse_args, create_logger, DEVICE, fix_random_seed -from callbacks import BaseCallback +from callbacks import BaseCallback, EvalCallback, ValidationCallback from dataset import BaseDataset from dataloader import BaseDataloader from loss import BaseLoss from models import BaseModel from optimizer import BaseOptimizer +from infer import inference import copy import json @@ -59,6 +60,7 @@ def train(dataloader, model, optimizer, loss_function, callback, epoch_cnt=None, current_metric = batch_[best_metric] best_checkpoint = copy.deepcopy(model.state_dict()) best_epoch = epoch_num + break epoch_num += 1 logger.debug('Training procedure has been finished!') @@ -124,7 +126,7 @@ def main(): logger.debug('Everything is ready for training process!') # Train process - _ = train( + best_model_checkpoint = train( dataloader=train_dataloader, model=model, optimizer=optimizer, @@ -135,6 +137,27 @@ def main(): best_metric=config.get('best_metric') ) + eval_model = BaseModel.create_from_config(config['model'], **dataset.meta).to(DEVICE) + eval_model.load_state_dict(best_model_checkpoint) + + for cl in callback._callbacks: + if isinstance(cl, EvalCallback): + metrics = cl._metrics + pred_prefix = cl._pred_prefix + labels_prefix = cl._labels_prefix + break + else: + for cl in callback._callbacks: + if isinstance(cl, ValidationCallback): + metrics = cl._metrics + pred_prefix = cl._pred_prefix + labels_prefix = cl._labels_prefix + break + else: + assert False + + inference(eval_dataloader, eval_model, metrics, pred_prefix, labels_prefix) + logger.debug('Saving model...') checkpoint_path = '../checkpoints/{}_final_state.pth'.format(config['experiment_name']) torch.save(model.state_dict(), checkpoint_path) From 254a16000be9382f45a357de51ebf5a67bb18edf Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Wed, 8 Jan 2025 23:37:26 +0700 Subject: [PATCH 09/15] fix configs --- configs/train/bert4rec_train_config_all_rank.json | 12 ++++++++++++ configs/train/bert4rec_train_config_popular.json | 12 ++++++++++++ configs/train/bert4rec_train_grid_config.json | 10 ++++++---- modeling/train.py | 1 - 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json index 11097be0..b13ab077 100644 --- a/configs/train/bert4rec_train_config_all_rank.json +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -105,6 +105,18 @@ "recall@20": { "type": "recall", "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 } } }, diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json index db7ab506..36bf9aec 100644 --- a/configs/train/bert4rec_train_config_popular.json +++ b/configs/train/bert4rec_train_config_popular.json @@ -106,6 +106,18 @@ "recall@20": { "type": "recall", "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 } } }, diff --git a/configs/train/bert4rec_train_grid_config.json b/configs/train/bert4rec_train_grid_config.json index f9cb31cc..5a64e7b2 100644 --- a/configs/train/bert4rec_train_grid_config.json +++ b/configs/train/bert4rec_train_grid_config.json @@ -1,10 +1,10 @@ { "start_from": 0, - "experiment_name": "bert4rec_beauty_grid", - "best_metric": "validation/ndcg@20", + "experiment_name": "bert4rec_grid_all_rank_beauty", + "best_metric": "eval/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -15,6 +15,8 @@ "dataset_params": { "samplers": { "mask_prob": [ + 0.1, + 0.2, 0.3, 0.4, 0.5, @@ -45,7 +47,7 @@ } }, "model": { - "type": "bert4rec", + "type": "bert4rec_all_rank", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", diff --git a/modeling/train.py b/modeling/train.py index 1090d723..ef95a664 100644 --- a/modeling/train.py +++ b/modeling/train.py @@ -60,7 +60,6 @@ def train(dataloader, model, optimizer, loss_function, callback, epoch_cnt=None, current_metric = batch_[best_metric] best_checkpoint = copy.deepcopy(model.state_dict()) best_epoch = epoch_num - break epoch_num += 1 logger.debug('Training procedure has been finished!') From ac68266617b3300093016fcef3c3c72e701f27a4 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Mon, 13 Jan 2025 18:04:55 +0700 Subject: [PATCH 10/15] fix bert4rec in batch --- .../train/bert4rec_train_config_in_batch.json | 169 ++++++++++++++++++ configs/train/bert4rec_train_grid_config.json | 9 +- .../train/sasrec_train_config_in_batch.json | 167 +++++++++++++++++ configs/train/sasrec_train_grid_config.json | 6 +- modeling/models/bert4rec_in_batch.py | 2 +- 5 files changed, 345 insertions(+), 8 deletions(-) create mode 100644 configs/train/bert4rec_train_config_in_batch.json create mode 100644 configs/train/sasrec_train_config_in_batch.json diff --git a/configs/train/bert4rec_train_config_in_batch.json b/configs/train/bert4rec_train_config_in_batch.json new file mode 100644 index 00000000..52f90f74 --- /dev/null +++ b/configs/train/bert4rec_train_config_in_batch.json @@ -0,0 +1,169 @@ +{ + "experiment_name": "bert4rec_in_batch_beauty", + "best_metric": "eval/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.7, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_in_batch", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "ce", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_grid_config.json b/configs/train/bert4rec_train_grid_config.json index 5a64e7b2..88a658f3 100644 --- a/configs/train/bert4rec_train_grid_config.json +++ b/configs/train/bert4rec_train_grid_config.json @@ -1,7 +1,7 @@ { "start_from": 0, - "experiment_name": "bert4rec_grid_all_rank_beauty", - "best_metric": "eval/ndcg@20", + "experiment_name": "bert4rec_grid_in_batch_beauty", + "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", "path_to_data_dir": "../data/sasrec_in_batch", @@ -22,7 +22,8 @@ 0.5, 0.6, 0.7, - 0.8 + 0.8, + 0.9 ] } }, @@ -47,7 +48,7 @@ } }, "model": { - "type": "bert4rec_all_rank", + "type": "bert4rec_in_batch", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", diff --git a/configs/train/sasrec_train_config_in_batch.json b/configs/train/sasrec_train_config_in_batch.json new file mode 100644 index 00000000..0fff79dd --- /dev/null +++ b/configs/train/sasrec_train_config_in_batch.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_in_batch_beauty", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_in_batch", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_grid_config.json b/configs/train/sasrec_train_grid_config.json index 2eb99b8a..2b6f5623 100644 --- a/configs/train/sasrec_train_grid_config.json +++ b/configs/train/sasrec_train_grid_config.json @@ -1,10 +1,10 @@ { "start_from": 0, - "experiment_name": "sasrec_beauty_grid", + "experiment_name": "sasrec_grid_in_batch_beauty", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -35,7 +35,7 @@ } }, "model": { - "type": "sasrec", + "type": "sasrec_in_batch", "sequence_prefix": "item", "positive_prefix": "positive", "negative_prefix": "negative", diff --git a/modeling/models/bert4rec_in_batch.py b/modeling/models/bert4rec_in_batch.py index e5cddb48..27671843 100644 --- a/modeling/models/bert4rec_in_batch.py +++ b/modeling/models/bert4rec_in_batch.py @@ -100,7 +100,7 @@ def forward(self, inputs): needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1=batch_size + 1) # needed_labels = all_sample_labels[labels_mask] # (non_zero_events) - needed_labels = torch.zeros(len(needed_logits), dtype=torch.long) # (non_zero_events) + needed_labels = torch.zeros(len(needed_logits), dtype=torch.long, device=needed_logits.device) # (non_zero_events) return {'logits': needed_logits, 'labels.ids': needed_labels} else: From 71eac21cc2395c8f16516e37f5d6e71966e40914 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Wed, 5 Feb 2025 21:51:12 +0700 Subject: [PATCH 11/15] impement sasrec_ce loss and bert4rec_sasrec loss --- .../inference/bert4rec_inference_config.json | 6 +- .../train/bert4rec_train_config_all_rank.json | 6 +- .../train/bert4rec_train_config_in_batch.json | 6 +- .../train/bert4rec_train_config_popular.json | 6 +- .../train/bert4rec_train_config_random.json | 170 ++++++++++++++++++ configs/train/bert4rec_train_grid_config.json | 5 +- .../train/sasrec_train_config_all_rank.json | 2 +- .../train/sasrec_train_config_in_batch.json | 2 +- configs/train/sasrec_train_config_random.json | 168 +++++++++++++++++ configs/train/sasrec_train_grid_config.json | 5 +- modeling/loss/base.py | 60 +++++++ modeling/models/sasrec.py | 6 +- 12 files changed, 421 insertions(+), 21 deletions(-) create mode 100644 configs/train/bert4rec_train_config_random.json create mode 100644 configs/train/sasrec_train_config_random.json diff --git a/configs/inference/bert4rec_inference_config.json b/configs/inference/bert4rec_inference_config.json index 03d75f3c..4bb8ce5c 100644 --- a/configs/inference/bert4rec_inference_config.json +++ b/configs/inference/bert4rec_inference_config.json @@ -1,10 +1,10 @@ { "pred_prefix": "logits", "label_prefix": "labels", - "experiment_name": "bert4rec_beauty_grid_0-5_0-1__", + "experiment_name": "bert4rec_all_rank_beauty", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -34,7 +34,7 @@ } }, "model": { - "type": "bert4rec", + "type": "bert4rec_all_rank", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json index b13ab077..c5a4d527 100644 --- a/configs/train/bert4rec_train_config_all_rank.json +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -1,13 +1,13 @@ { "experiment_name": "bert4rec_all_rank_beauty", - "best_metric": "eval/ndcg@20", + "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.7, + "mask_prob": 0.6, "type": "masked_item_prediction", "negative_sampler_type": "random" } @@ -42,7 +42,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.1, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 diff --git a/configs/train/bert4rec_train_config_in_batch.json b/configs/train/bert4rec_train_config_in_batch.json index 52f90f74..7d671b77 100644 --- a/configs/train/bert4rec_train_config_in_batch.json +++ b/configs/train/bert4rec_train_config_in_batch.json @@ -1,13 +1,13 @@ { "experiment_name": "bert4rec_in_batch_beauty", - "best_metric": "eval/ndcg@20", + "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.7, + "mask_prob": 0.4, "type": "masked_item_prediction", "negative_sampler_type": "random" } @@ -42,7 +42,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.2, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json index 36bf9aec..d4027249 100644 --- a/configs/train/bert4rec_train_config_popular.json +++ b/configs/train/bert4rec_train_config_popular.json @@ -1,13 +1,13 @@ { "experiment_name": "bert4rec_popular_beauty", - "best_metric": "eval/ndcg@20", + "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.7, + "mask_prob": 0.6, "num_negatives_train": 100, "type": "masked_item_prediction", "negative_sampler_type": "random_by_popularity" @@ -43,7 +43,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.1, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 diff --git a/configs/train/bert4rec_train_config_random.json b/configs/train/bert4rec_train_config_random.json new file mode 100644 index 00000000..62f91e48 --- /dev/null +++ b/configs/train/bert4rec_train_config_random.json @@ -0,0 +1,170 @@ +{ + "experiment_name": "bert4rec_popular_beauty", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.6, + "num_negatives_train": 100, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_popular", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.1, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "ce", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_grid_config.json b/configs/train/bert4rec_train_grid_config.json index 88a658f3..c3aa75bd 100644 --- a/configs/train/bert4rec_train_grid_config.json +++ b/configs/train/bert4rec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "bert4rec_grid_in_batch_beauty", + "experiment_name": "bert4rec_grid_random_beauty", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -8,6 +8,7 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { + "num_negatives_train": 100, "type": "masked_item_prediction", "negative_sampler_type": "random" } @@ -48,7 +49,7 @@ } }, "model": { - "type": "bert4rec_in_batch", + "type": "bert4rec_popular", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", diff --git a/configs/train/sasrec_train_config_all_rank.json b/configs/train/sasrec_train_config_all_rank.json index 7bae9db9..fa0a17cc 100644 --- a/configs/train/sasrec_train_config_all_rank.json +++ b/configs/train/sasrec_train_config_all_rank.json @@ -41,7 +41,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.5, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 diff --git a/configs/train/sasrec_train_config_in_batch.json b/configs/train/sasrec_train_config_in_batch.json index 0fff79dd..173b2975 100644 --- a/configs/train/sasrec_train_config_in_batch.json +++ b/configs/train/sasrec_train_config_in_batch.json @@ -41,7 +41,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.1, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 diff --git a/configs/train/sasrec_train_config_random.json b/configs/train/sasrec_train_config_random.json new file mode 100644 index 00000000..78fb4224 --- /dev/null +++ b/configs/train/sasrec_train_config_random.json @@ -0,0 +1,168 @@ +{ + "experiment_name": "sasrec_random_beauty", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "num_negatives_train": 100, + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_popular", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.3, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_grid_config.json b/configs/train/sasrec_train_grid_config.json index 2b6f5623..fda935fc 100644 --- a/configs/train/sasrec_train_grid_config.json +++ b/configs/train/sasrec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "sasrec_grid_in_batch_beauty", + "experiment_name": "sasrec_grid_random_beauty", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -8,6 +8,7 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { + "num_negatives_train": 100, "type": "next_item_prediction", "negative_sampler_type": "random" } @@ -35,7 +36,7 @@ } }, "model": { - "type": "sasrec_in_batch", + "type": "sasrec_popular", "sequence_prefix": "item", "positive_prefix": "positive", "negative_prefix": "negative", diff --git a/modeling/loss/base.py b/modeling/loss/base.py index ef8ba88c..b3854565 100644 --- a/modeling/loss/base.py +++ b/modeling/loss/base.py @@ -105,6 +105,36 @@ def forward(self, inputs): return loss +class CrossEntropyLossSasrec(TorchLoss, config_name='sasrec_ce'): + + def __init__( + self, + positive_prefix, + negative_prefix, + output_prefix=None + ): + super().__init__() + self._positive_prefix = positive_prefix + self._negative_prefix = negative_prefix + self._output_prefix = output_prefix + + self._loss = nn.CrossEntropyLoss() + + def forward(self, inputs): + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) + assert positive_scores.shape[0] == negative_scores.shape[0] + + all_logits = torch.cat((positive_scores, negative_scores), dim=1) + all_labels = torch.zeros(len(all_logits), dtype=torch.long, device=all_logits.device) + assert all_logits.shape[0] == all_labels.shape[0] + + loss = self._loss(all_logits, all_labels) # (1) + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss + class BinaryCrossEntropyLoss(TorchLoss, config_name='bce'): @@ -269,6 +299,36 @@ def forward(self, inputs): return loss +class Bert4recSASRecLoss(TorchLoss, config_name='bert4rec_sasrec'): + + def __init__(self, predictions_prefix, labels_prefix, output_prefix=None): + super().__init__() + self._pred_prefix = predictions_prefix + self._labels_prefix = labels_prefix + self._output_prefix = output_prefix + + def forward(self, inputs): + all_logits = inputs[self._pred_prefix] # (all_items, num_classes) + all_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_items) + assert all_logits.shape[0] == all_labels.shape[0] + + # Extract positive scores using gather + positive_scores = all_logits.gather(1, all_labels.unsqueeze(1)).squeeze(1) + # Create a mask to exclude the positive indices + mask = torch.arange(all_logits.size(1), device=all_logits.device) != all_labels.unsqueeze(1) + # Apply the mask and reshape to get negative scores + negative_scores = all_logits[mask].view(all_logits.size(0), -1) + assert positive_scores.shape[0] == negative_scores.shape[0] + + positive_loss = torch.log(nn.functional.sigmoid(positive_scores) + 1e-9).sum(dim=-1) # (x) + negative_loss = torch.log(1.0 - nn.functional.sigmoid(negative_scores) + 1e-9).sum(dim=-1) # (x) + loss = positive_loss + negative_loss # (x) + loss = -loss.sum() # (1) + + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss class SamplesSoftmaxLoss(TorchLoss, config_name='sampled_softmax'): diff --git a/modeling/models/sasrec.py b/modeling/models/sasrec.py index 869ef653..9e1cd268 100644 --- a/modeling/models/sasrec.py +++ b/modeling/models/sasrec.py @@ -295,17 +295,17 @@ def forward(self, inputs): if self.training: # training mode # TODO: move 'negative' to config negative_items = inputs['negative.ids'] # (num_negatives) - negative_embeddings = self._item_embeddings.weight[negative_items] # (num_negatives, embedding_dim) + negative_embeddings = self._item_embeddings.weight[negative_items] # (batch_size*num_negatives, embedding_dim) negative_embeddings = negative_embeddings.reshape(len(all_sample_lengths), int(negative_embeddings.size(0) / len(all_sample_lengths)), negative_embeddings.size(1)) # (batch_size, num_negatives, embedding_dim) negative_embeddings = torch.repeat_interleave(negative_embeddings, all_sample_lengths, dim=0) # (all_batch_events, num_negatives, embedding_dim) all_sample_embeddings = embeddings[mask] # (all_batch_events, embedding_dim) - negative_scores = torch.einsum("bd,bnd->bn", all_sample_embeddings, negative_embeddings) # (non_zero_events, num_negatives) + negative_scores = torch.einsum("bd,bnd->bn", all_sample_embeddings, negative_embeddings) # (all_batch_events, num_negatives) # negative_scores = all_sample_embeddings @ negative_embeddings.T all_positive_sample_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) all_positive_sample_embeddings = self._item_embeddings.weight[ - all_positive_sample_events] # (non_zero_events, embedding_dim) + all_positive_sample_events] # (all_batch_events, embedding_dim) positive_scores = (all_sample_embeddings * all_positive_sample_embeddings).sum(dim=-1) From 8ad4f73c13fe486ebba3c0c9f58507427f11c196 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Tue, 11 Feb 2025 15:32:57 +0100 Subject: [PATCH 12/15] fix sasrec all_rank positive scores shape --- configs/train/bert4rec_train_grid_config.json | 9 ++++----- configs/train/sasrec_train_grid_config.json | 7 +++---- modeling/models/sasrec.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/configs/train/bert4rec_train_grid_config.json b/configs/train/bert4rec_train_grid_config.json index c3aa75bd..64bcf098 100644 --- a/configs/train/bert4rec_train_grid_config.json +++ b/configs/train/bert4rec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "bert4rec_grid_random_beauty", + "experiment_name": "bert4rec_grid_in_batch_beauty_loss_sasrec", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -8,9 +8,8 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "num_negatives_train": 100, "type": "masked_item_prediction", - "negative_sampler_type": "random" + "negative_sampler_type": "random_by_popularity" } }, "dataset_params": { @@ -49,7 +48,7 @@ } }, "model": { - "type": "bert4rec_popular", + "type": "bert4rec_in_batch", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", @@ -86,7 +85,7 @@ "type": "composite", "losses": [ { - "type": "ce", + "type": "bert4rec_sasrec", "predictions_prefix": "logits", "labels_prefix": "labels", "output_prefix": "downstream_loss" diff --git a/configs/train/sasrec_train_grid_config.json b/configs/train/sasrec_train_grid_config.json index fda935fc..70fba476 100644 --- a/configs/train/sasrec_train_grid_config.json +++ b/configs/train/sasrec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "sasrec_grid_random_beauty", + "experiment_name": "sasrec_grid_all_rank_beauty_loss_ce", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -8,7 +8,6 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "num_negatives_train": 100, "type": "next_item_prediction", "negative_sampler_type": "random" } @@ -36,7 +35,7 @@ } }, "model": { - "type": "sasrec_popular", + "type": "sasrec_all_rank", "sequence_prefix": "item", "positive_prefix": "positive", "negative_prefix": "negative", @@ -75,7 +74,7 @@ "type": "composite", "losses": [ { - "type": "sasrec", + "type": "sasrec_ce", "positive_prefix": "positive_scores", "negative_prefix": "negative_scores", "output_prefix": "downstream_loss" diff --git a/modeling/models/sasrec.py b/modeling/models/sasrec.py index 9e1cd268..d7e9a6d1 100644 --- a/modeling/models/sasrec.py +++ b/modeling/models/sasrec.py @@ -194,7 +194,7 @@ def forward(self, inputs): input=all_scores, dim=1, index=all_positive_sample_events[..., None] - ) # (all_batch_items, 1) + ).squeeze() # (all_batch_items,) sample_ids, _ = create_masked_tensor( data=all_sample_events, From e3133fe8b9b14385f0fe3e627fef13eae8e4d2ec Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Tue, 11 Feb 2025 15:59:21 +0100 Subject: [PATCH 13/15] fix sasrec in batch --- configs/train/sasrec_train_grid_config.json | 4 ++-- modeling/models/sasrec.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/train/sasrec_train_grid_config.json b/configs/train/sasrec_train_grid_config.json index 70fba476..fbc01f9d 100644 --- a/configs/train/sasrec_train_grid_config.json +++ b/configs/train/sasrec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "sasrec_grid_all_rank_beauty_loss_ce", + "experiment_name": "sasrec_grid_in_batch_beauty_loss_ce", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -35,7 +35,7 @@ } }, "model": { - "type": "sasrec_all_rank", + "type": "sasrec_in_batch", "sequence_prefix": "item", "positive_prefix": "positive", "negative_prefix": "negative", diff --git a/modeling/models/sasrec.py b/modeling/models/sasrec.py index d7e9a6d1..578ffc6b 100644 --- a/modeling/models/sasrec.py +++ b/modeling/models/sasrec.py @@ -406,11 +406,12 @@ def forward(self, inputs): in_batch_negative_ids ) # (batch_size, embedding_dim) - in_batch_negative_embeddings = in_batch_queries_embeddings @ in_batch_negative_embeddings.T + in_batch_negative_scores = in_batch_queries_embeddings @ in_batch_negative_embeddings.T + in_batch_positive_scores = (in_batch_queries_embeddings * in_batch_positive_embeddings).sum(dim=-1) return { - 'positive_scores': in_batch_positive_embeddings, - 'negative_scores': in_batch_negative_embeddings + 'positive_scores': in_batch_positive_scores, + 'negative_scores': in_batch_negative_scores } else: # eval mode last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) From 2f1cdf19b584c8404602109c4d6f787f182563af Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Tue, 25 Mar 2025 23:47:15 +0700 Subject: [PATCH 14/15] implement log q and adaptive --- .../train/bert4rec_train_config_all_rank.json | 8 +- .../train/bert4rec_train_config_in_batch.json | 8 +- .../train/bert4rec_train_config_popular.json | 8 +- .../train/bert4rec_train_config_random.json | 8 +- configs/train/bert4rec_train_grid_config.json | 31 ++-- .../train/sasrec_train_config_all_rank.json | 6 +- .../train/sasrec_train_config_in_batch.json | 6 +- ...sasrec_train_config_in_batch_adaptive.json | 167 ++++++++++++++++++ .../sasrec_train_config_in_batch_log_q.json | 167 ++++++++++++++++++ .../train/sasrec_train_config_popular.json | 6 +- configs/train/sasrec_train_config_random.json | 6 +- configs/train/sasrec_train_grid_config.json | 20 +-- .../dataset/samplers/next_item_prediction.py | 44 ++++- modeling/loss/base.py | 103 +++++++++++ modeling/models/sasrec.py | 26 +-- 15 files changed, 543 insertions(+), 71 deletions(-) create mode 100644 configs/train/sasrec_train_config_in_batch_adaptive.json create mode 100644 configs/train/sasrec_train_config_in_batch_log_q.json diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json index c5a4d527..077cc6bd 100644 --- a/configs/train/bert4rec_train_config_all_rank.json +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -1,5 +1,5 @@ { - "experiment_name": "bert4rec_all_rank_beauty", + "experiment_name": "bert4rec_all_rank_beauty_bce_0-3_0-2", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -7,7 +7,7 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.6, + "mask_prob": 0.3, "type": "masked_item_prediction", "negative_sampler_type": "random" } @@ -42,7 +42,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.1, + "dropout": 0.2, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -59,7 +59,7 @@ "type": "composite", "losses": [ { - "type": "ce", + "type": "bert4rec_sasrec", "predictions_prefix": "logits", "labels_prefix": "labels", "output_prefix": "downstream_loss", diff --git a/configs/train/bert4rec_train_config_in_batch.json b/configs/train/bert4rec_train_config_in_batch.json index 7d671b77..8a83211b 100644 --- a/configs/train/bert4rec_train_config_in_batch.json +++ b/configs/train/bert4rec_train_config_in_batch.json @@ -1,5 +1,5 @@ { - "experiment_name": "bert4rec_in_batch_beauty", + "experiment_name": "bert4rec_in_batch_beauty_bce_0-3_0-5", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -7,7 +7,7 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.4, + "mask_prob": 0.3, "type": "masked_item_prediction", "negative_sampler_type": "random" } @@ -42,7 +42,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.2, + "dropout": 0.5, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -59,7 +59,7 @@ "type": "composite", "losses": [ { - "type": "ce", + "type": "bert4rec_sasrec", "predictions_prefix": "logits", "labels_prefix": "labels", "output_prefix": "downstream_loss", diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json index d4027249..2a05db02 100644 --- a/configs/train/bert4rec_train_config_popular.json +++ b/configs/train/bert4rec_train_config_popular.json @@ -1,5 +1,5 @@ { - "experiment_name": "bert4rec_popular_beauty", + "experiment_name": "bert4rec_popular_beauty_bce_0-3_0-4", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -7,7 +7,7 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.6, + "mask_prob": 0.3, "num_negatives_train": 100, "type": "masked_item_prediction", "negative_sampler_type": "random_by_popularity" @@ -43,7 +43,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.1, + "dropout": 0.4, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -60,7 +60,7 @@ "type": "composite", "losses": [ { - "type": "ce", + "type": "bert4rec_sasrec", "predictions_prefix": "logits", "labels_prefix": "labels", "output_prefix": "downstream_loss", diff --git a/configs/train/bert4rec_train_config_random.json b/configs/train/bert4rec_train_config_random.json index 62f91e48..a72536f1 100644 --- a/configs/train/bert4rec_train_config_random.json +++ b/configs/train/bert4rec_train_config_random.json @@ -1,5 +1,5 @@ { - "experiment_name": "bert4rec_popular_beauty", + "experiment_name": "bert4rec_random_beauty_bce_0-5_0-4", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -7,7 +7,7 @@ "name": "Beauty", "max_sequence_length": 50, "samplers": { - "mask_prob": 0.6, + "mask_prob": 0.5, "num_negatives_train": 100, "type": "masked_item_prediction", "negative_sampler_type": "random" @@ -43,7 +43,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.1, + "dropout": 0.4, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -60,7 +60,7 @@ "type": "composite", "losses": [ { - "type": "ce", + "type": "bert4rec_sasrec", "predictions_prefix": "logits", "labels_prefix": "labels", "output_prefix": "downstream_loss", diff --git a/configs/train/bert4rec_train_grid_config.json b/configs/train/bert4rec_train_grid_config.json index 64bcf098..dedeb280 100644 --- a/configs/train/bert4rec_train_grid_config.json +++ b/configs/train/bert4rec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "bert4rec_grid_in_batch_beauty_loss_sasrec", + "experiment_name": "bert4rec_popular_beauty_0-4_0-2_num_neg", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -14,16 +14,13 @@ }, "dataset_params": { "samplers": { + "num_negatives_train": [ + 500, + 1000, + 5000 + ], "mask_prob": [ - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9 + 0.4 ] } }, @@ -48,7 +45,7 @@ } }, "model": { - "type": "bert4rec_in_batch", + "type": "bert4rec_popular", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", @@ -62,14 +59,8 @@ }, "model_params": { "dropout": [ - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8] + 0.2 + ] }, "optimizer": { "type": "basic", @@ -85,7 +76,7 @@ "type": "composite", "losses": [ { - "type": "bert4rec_sasrec", + "type": "ce", "predictions_prefix": "logits", "labels_prefix": "labels", "output_prefix": "downstream_loss" diff --git a/configs/train/sasrec_train_config_all_rank.json b/configs/train/sasrec_train_config_all_rank.json index fa0a17cc..2db3f8ac 100644 --- a/configs/train/sasrec_train_config_all_rank.json +++ b/configs/train/sasrec_train_config_all_rank.json @@ -1,5 +1,5 @@ { - "experiment_name": "sasrec_all_rank_beauty", + "experiment_name": "sasrec_all_rank_beauty_loss_ce_0-7", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -41,7 +41,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.5, + "dropout": 0.7, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -58,7 +58,7 @@ "type": "composite", "losses": [ { - "type": "sasrec", + "type": "sasrec_ce", "positive_prefix": "positive_scores", "negative_prefix": "negative_scores", "output_prefix": "downstream_loss" diff --git a/configs/train/sasrec_train_config_in_batch.json b/configs/train/sasrec_train_config_in_batch.json index 173b2975..9b7f753f 100644 --- a/configs/train/sasrec_train_config_in_batch.json +++ b/configs/train/sasrec_train_config_in_batch.json @@ -1,5 +1,5 @@ { - "experiment_name": "sasrec_in_batch_beauty", + "experiment_name": "sasrec_in_batch_beauty_loss_ce_0-4", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -41,7 +41,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.1, + "dropout": 0.4, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -58,7 +58,7 @@ "type": "composite", "losses": [ { - "type": "sasrec", + "type": "sasrec_ce", "positive_prefix": "positive_scores", "negative_prefix": "negative_scores", "output_prefix": "downstream_loss" diff --git a/configs/train/sasrec_train_config_in_batch_adaptive.json b/configs/train/sasrec_train_config_in_batch_adaptive.json new file mode 100644 index 00000000..5028390f --- /dev/null +++ b/configs/train/sasrec_train_config_in_batch_adaptive.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_in_batch_beauty_loss_ce_adaptive", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_in_batch", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce_adaptive", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_in_batch_log_q.json b/configs/train/sasrec_train_config_in_batch_log_q.json new file mode 100644 index 00000000..4c74030e --- /dev/null +++ b/configs/train/sasrec_train_config_in_batch_log_q.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_in_batch_beauty_loss_ce_log_q", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_in_batch", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce_log_q", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_popular.json b/configs/train/sasrec_train_config_popular.json index a5436e2e..f968ff7c 100644 --- a/configs/train/sasrec_train_config_popular.json +++ b/configs/train/sasrec_train_config_popular.json @@ -1,5 +1,5 @@ { - "experiment_name": "sasrec_popular_beauty", + "experiment_name": "sasrec_popular_beauty_loss_ce_0-6", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -42,7 +42,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.6, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -59,7 +59,7 @@ "type": "composite", "losses": [ { - "type": "sasrec", + "type": "sasrec_ce", "positive_prefix": "positive_scores", "negative_prefix": "negative_scores", "output_prefix": "downstream_loss" diff --git a/configs/train/sasrec_train_config_random.json b/configs/train/sasrec_train_config_random.json index 78fb4224..890bbb24 100644 --- a/configs/train/sasrec_train_config_random.json +++ b/configs/train/sasrec_train_config_random.json @@ -1,5 +1,5 @@ { - "experiment_name": "sasrec_random_beauty", + "experiment_name": "sasrec_random_beauty_loss_ce_0-5", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -42,7 +42,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.3, + "dropout": 0.5, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 @@ -59,7 +59,7 @@ "type": "composite", "losses": [ { - "type": "sasrec", + "type": "sasrec_ce", "positive_prefix": "positive_scores", "negative_prefix": "negative_scores", "output_prefix": "downstream_loss" diff --git a/configs/train/sasrec_train_grid_config.json b/configs/train/sasrec_train_grid_config.json index fbc01f9d..7ccb7fc6 100644 --- a/configs/train/sasrec_train_grid_config.json +++ b/configs/train/sasrec_train_grid_config.json @@ -1,6 +1,6 @@ { "start_from": 0, - "experiment_name": "sasrec_grid_in_batch_beauty_loss_ce", + "experiment_name": "sasrec_grid_random_beauty_num_neg", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", @@ -13,6 +13,11 @@ } }, "dataset_params": { + "num_negatives_train": [ + 100, + 500, + 1000 + ] }, "dataloader": { "train": { @@ -35,7 +40,7 @@ } }, "model": { - "type": "sasrec_in_batch", + "type": "sasrec_popular", "sequence_prefix": "item", "positive_prefix": "positive", "negative_prefix": "negative", @@ -50,15 +55,8 @@ }, "model_params": { "dropout": [ - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9] + 0.5 + ] }, "optimizer": { "type": "basic", diff --git a/modeling/dataset/samplers/next_item_prediction.py b/modeling/dataset/samplers/next_item_prediction.py index c141b065..c4290b0c 100644 --- a/modeling/dataset/samplers/next_item_prediction.py +++ b/modeling/dataset/samplers/next_item_prediction.py @@ -1,7 +1,37 @@ from dataset.samplers.base import TrainSampler, EvalSampler from dataset.negative_samplers.base import BaseNegativeSampler +import torch import copy +from collections import defaultdict +import numpy as np + + +class DatasetAnalyzer: + def __init__(self, dataset): + self.dataset = dataset + self.item_freq = None # Словарь для хранения частот + self.all_items_count = 0 + + def precompute_frequencies(self): + """Подсчет частот встречаемости item_id в dataset""" + freq = defaultdict(int) + + # Проходим по всем элементам dataset + for sample in self.dataset: + for item_id in sample['item.ids']: + freq[item_id] += 1 + self.all_items_count += len(sample['item.ids']) + + self.item_freq = freq + + def get_frequency(self, item_ids): + """Возвращает список частот для каждого item_id из списка item_ids""" + if not isinstance(item_ids, list): + raise TypeError("item_ids должен быть списком") + + # Для каждого item_id в списке возвращаем его частоту или 0, если его нет + return [self.item_freq.get(item_id, 0) for item_id in item_ids] class NextItemPredictionTrainSampler(TrainSampler, config_name='next_item_prediction'): @@ -13,6 +43,8 @@ def __init__(self, dataset, num_users, num_items, negative_sampler, num_negative self._num_items = num_items self._negative_sampler = negative_sampler self._num_negatives = num_negatives + self._frequency_counter = DatasetAnalyzer(dataset) + self._frequency_counter.precompute_frequencies() @classmethod def create_from_config(cls, config, **kwargs): @@ -26,6 +58,7 @@ def create_from_config(cls, config, **kwargs): num_negatives=config.get('num_negatives_train', 0) ) + # here add how often each element in batch is occur def __getitem__(self, index): sample = copy.deepcopy(self._dataset[index]) @@ -40,8 +73,17 @@ def __getitem__(self, index): 'item.ids': item_sequence, 'item.length': len(item_sequence), + 'item_counts.ids': self._frequency_counter.get_frequency(item_sequence), + 'item_counts.length': len(item_sequence), + 'positive.ids': next_item_sequence, - 'positive.length': len(next_item_sequence) + 'positive.length': len(next_item_sequence), + + 'positive_counts.ids': self._frequency_counter.get_frequency(next_item_sequence), + 'positive_counts.length': len(next_item_sequence), + + 'counts.ids': [self._frequency_counter.all_items_count], + 'counts.length': 1, } else: negative_sequence = self._negative_sampler.generate_negative_samples( diff --git a/modeling/loss/base.py b/modeling/loss/base.py index b3854565..873045ad 100644 --- a/modeling/loss/base.py +++ b/modeling/loss/base.py @@ -105,6 +105,7 @@ def forward(self, inputs): return loss + class CrossEntropyLossSasrec(TorchLoss, config_name='sasrec_ce'): def __init__( @@ -135,6 +136,108 @@ def forward(self, inputs): return loss +# positive_logits (all_samples) +# negative_logits (all_samples, num_negatives) +# all_counts () +# negative_counts (num_negatives) +def process_log_q(positive_logits, negative_logits, all_counts, negative_counts): + # Вычисляем поправку log(q) для негативных примеров + log_q = torch.log(negative_counts / all_counts) # (num_negatives) + adjusted_neg_logits = negative_logits - log_q # (all_samples, num_negatives) + + # Объединяем положительный и скорректированные негативные логиты + combined_logits = torch.cat([positive_logits.unsqueeze(1), adjusted_neg_logits], dim=1) # (all_samples, num_negatives + 1) + + loss = -torch.logsoftmax(combined_logits, dim=-1)[:, 0].mean() + + return loss + +class CrossEntropyLossSasrecLogQ(TorchLoss, config_name='sasrec_ce_log_q'): + + def __init__( + self, + positive_prefix, + negative_prefix, + output_prefix=None + ): + super().__init__() + self._positive_prefix = positive_prefix + self._negative_prefix = negative_prefix + self._output_prefix = output_prefix + + def forward(self, inputs): + # import code + # code.interact(local=locals()) + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) + assert positive_scores.shape[0] == negative_scores.shape[0] + all_counts = inputs['all_counts'] # () + negative_counts = inputs['negative_counts'] # (num_negatives) + + log_q = torch.log(negative_counts / all_counts) # (num_negatives) + negative_scores = negative_scores - log_q # (all_samples, num_negatives) + + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (all_samples, num_negatives + 1) + loss = -torch.log_softmax(all_logits, dim=-1)[:, 0].mean() # (1) + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss + +def log_softmax_without_self(logits: torch.Tensor, dim: int = -1) -> torch.Tensor: + """ + Вычисляет log-softmax, исключая текущий элемент из знаменателя. + """ + log_softmax_normal = torch.log_softmax(logits, dim=dim) + correction = torch.log1p(-torch.exp(log_softmax_normal)) + return log_softmax_normal - correction + +class CrossEntropyLossSasrecAdaptive(TorchLoss, config_name='sasrec_ce_adaptive'): + + def __init__( + self, + positive_prefix, + negative_prefix, + output_prefix=None + ): + super().__init__() + self._positive_prefix = positive_prefix + self._negative_prefix = negative_prefix + self._output_prefix = output_prefix + + def forward(self, inputs): + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) + assert positive_scores.shape[0] == negative_scores.shape[0] + all_counts = inputs['all_counts'] # () + negative_counts = inputs['negative_counts'] # (num_negatives) + positive_counts = inputs['positive_counts'] # (x) + + log_q = torch.log(negative_counts[None, :] / (all_counts - positive_counts[:, None])) # (x, num_negatives) + negative_scores = negative_scores - log_q # (x, num_negatives) + + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (x, num_negatives + 1) + loss = -log_softmax_without_self(all_logits, dim=-1)[:, 0].mean() # () + + positive_scores_sg = positive_scores.detach() # (x, 1) + negative_scores_sg = negative_scores.detach() # (x, num_negatives) + all_counts_sg = all_counts.detach() # () + negative_counts_sg = negative_counts.detach() # (num_negatives) + positive_counts_sg = positive_counts.detach() # (x) + + log_q_sg = torch.log(negative_counts_sg[None, :] / (all_counts_sg - positive_counts_sg[:, None])) # (x, num_negatives) + negative_scores_sg = negative_scores_sg - log_q_sg + negative_scores_sg = negative_scores_sg - torch.log(torch.tensor(negative_counts_sg.shape[0], device=negative_scores_sg.device)) + all_logits_sg = torch.cat((positive_scores_sg, negative_scores_sg), dim=1) # (x, num_negatives + 1) + loss_sg = -torch.log_softmax(all_logits_sg, dim=-1)[:, 0].mean() + + loss = (1 - loss_sg) * loss + + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss + class BinaryCrossEntropyLoss(TorchLoss, config_name='bce'): diff --git a/modeling/models/sasrec.py b/modeling/models/sasrec.py index 578ffc6b..a751910f 100644 --- a/modeling/models/sasrec.py +++ b/modeling/models/sasrec.py @@ -393,25 +393,29 @@ def forward(self, inputs): # positives in_batch_positive_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) - in_batch_positive_embeddings = self._item_embeddings( - in_batch_positive_events - ) # (all_batch_events, embedding_dim) + in_batch_positive_embeddings = self._item_embeddings(in_batch_positive_events) # (all_batch_events, embedding_dim) + in_batch_positive_counts = inputs['positive_counts.ids'] # negatives - batch_size = all_sample_lengths.shape[0] - random_ids = torch.randperm(in_batch_positive_events.shape[0]) - in_batch_negative_ids = in_batch_positive_events[random_ids][:batch_size] + batch_size = all_sample_lengths.shape[0] # scalar + random_ids = torch.randperm(in_batch_positive_events.shape[0]) # (batch_size) + in_batch_negative_ids = in_batch_positive_events[random_ids][:batch_size] # (batch_size) + in_batch_negative_counts = in_batch_positive_counts[random_ids][:batch_size] # (batch_size) + all_counts = inputs['counts.ids'][0] # scalar - in_batch_negative_embeddings = self._item_embeddings( - in_batch_negative_ids - ) # (batch_size, embedding_dim) + in_batch_negative_embeddings = self._item_embeddings(in_batch_negative_ids) # (batch_size, embedding_dim) in_batch_negative_scores = in_batch_queries_embeddings @ in_batch_negative_embeddings.T in_batch_positive_scores = (in_batch_queries_embeddings * in_batch_positive_embeddings).sum(dim=-1) + # import code + # code.interact(local=locals()) return { - 'positive_scores': in_batch_positive_scores, - 'negative_scores': in_batch_negative_scores + 'positive_scores': in_batch_positive_scores, # [i] + 'negative_scores': in_batch_negative_scores, # [i] + 'positive_counts': in_batch_positive_counts, + 'negative_counts': in_batch_negative_counts, # all + 'all_counts': all_counts # } else: # eval mode last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) From 19028ad2370942e48dcc757948321738233d81b0 Mon Sep 17 00:00:00 2001 From: Yaroslav Zhurba Date: Sat, 5 Apr 2025 19:01:09 +0700 Subject: [PATCH 15/15] fix adaptive loss --- modeling/loss/base.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/modeling/loss/base.py b/modeling/loss/base.py index 873045ad..6884d309 100644 --- a/modeling/loss/base.py +++ b/modeling/loss/base.py @@ -141,11 +141,9 @@ def forward(self, inputs): # all_counts () # negative_counts (num_negatives) def process_log_q(positive_logits, negative_logits, all_counts, negative_counts): - # Вычисляем поправку log(q) для негативных примеров log_q = torch.log(negative_counts / all_counts) # (num_negatives) adjusted_neg_logits = negative_logits - log_q # (all_samples, num_negatives) - # Объединяем положительный и скорректированные негативные логиты combined_logits = torch.cat([positive_logits.unsqueeze(1), adjusted_neg_logits], dim=1) # (all_samples, num_negatives + 1) loss = -torch.logsoftmax(combined_logits, dim=-1)[:, 0].mean() @@ -189,7 +187,7 @@ def log_softmax_without_self(logits: torch.Tensor, dim: int = -1) -> torch.Tenso Вычисляет log-softmax, исключая текущий элемент из знаменателя. """ log_softmax_normal = torch.log_softmax(logits, dim=dim) - correction = torch.log1p(-torch.exp(log_softmax_normal)) + correction = torch.log(-torch.expm1(log_softmax_normal) + 1e-8) return log_softmax_normal - correction class CrossEntropyLossSasrecAdaptive(TorchLoss, config_name='sasrec_ce_adaptive'): @@ -206,32 +204,30 @@ def __init__( self._output_prefix = output_prefix def forward(self, inputs): - positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + # import code + # code.interact(local=locals()) + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) negative_scores = inputs[self._negative_prefix] # (x, num_negatives) assert positive_scores.shape[0] == negative_scores.shape[0] all_counts = inputs['all_counts'] # () negative_counts = inputs['negative_counts'] # (num_negatives) positive_counts = inputs['positive_counts'] # (x) + batch_size = negative_scores.shape[-1] + log_q = torch.log(negative_counts[None, :] / (all_counts - positive_counts[:, None])) # (x, num_negatives) negative_scores = negative_scores - log_q # (x, num_negatives) - all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (x, num_negatives + 1) - loss = -log_softmax_without_self(all_logits, dim=-1)[:, 0].mean() # () - - positive_scores_sg = positive_scores.detach() # (x, 1) - negative_scores_sg = negative_scores.detach() # (x, num_negatives) - all_counts_sg = all_counts.detach() # () - negative_counts_sg = negative_counts.detach() # (num_negatives) - positive_counts_sg = positive_counts.detach() # (x) + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (all_samples, num_negatives + 1) + loss = -torch.log_softmax(all_logits, dim=-1)[:, 0] # (x) - log_q_sg = torch.log(negative_counts_sg[None, :] / (all_counts_sg - positive_counts_sg[:, None])) # (x, num_negatives) - negative_scores_sg = negative_scores_sg - log_q_sg - negative_scores_sg = negative_scores_sg - torch.log(torch.tensor(negative_counts_sg.shape[0], device=negative_scores_sg.device)) - all_logits_sg = torch.cat((positive_scores_sg, negative_scores_sg), dim=1) # (x, num_negatives + 1) - loss_sg = -torch.log_softmax(all_logits_sg, dim=-1)[:, 0].mean() + negative_scores = negative_scores - log_q - torch.log(torch.ones_like(negative_scores) * (batch_size - 1)) # (x, num_negatives) + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (all_samples, num_negatives + 1) + prob = torch.softmax(all_logits, dim=-1)[:, 0] # (x) + multiplier = (1.0 - prob).detach() # (x) - loss = (1 - loss_sg) * loss + loss *= multiplier + loss = loss.mean() if self._output_prefix is not None: inputs[self._output_prefix] = loss.cpu().item()