diff --git a/configs/inference/bert4rec_inference_config.json b/configs/inference/bert4rec_inference_config.json index 03d75f3c..4bb8ce5c 100644 --- a/configs/inference/bert4rec_inference_config.json +++ b/configs/inference/bert4rec_inference_config.json @@ -1,10 +1,10 @@ { "pred_prefix": "logits", "label_prefix": "labels", - "experiment_name": "bert4rec_beauty_grid_0-5_0-1__", + "experiment_name": "bert4rec_all_rank_beauty", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -34,7 +34,7 @@ } }, "model": { - "type": "bert4rec", + "type": "bert4rec_all_rank", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", diff --git a/configs/train/bert4rec_train_config.json b/configs/train/bert4rec_train_config.json index 8e23f22c..bdf343ec 100644 --- a/configs/train/bert4rec_train_config.json +++ b/configs/train/bert4rec_train_config.json @@ -3,7 +3,7 @@ "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -41,7 +41,7 @@ "num_heads": 2, "num_layers": 2, "dim_feedforward": 256, - "dropout": 0.2, + "dropout": 0.3, "activation": "gelu", "layer_norm_eps": 1e-9, "initializer_range": 0.02 diff --git a/configs/train/bert4rec_train_config_all_rank.json b/configs/train/bert4rec_train_config_all_rank.json new file mode 100644 index 00000000..077cc6bd --- /dev/null +++ b/configs/train/bert4rec_train_config_all_rank.json @@ -0,0 +1,169 @@ +{ + "experiment_name": "bert4rec_all_rank_beauty_bce_0-3_0-2", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.3, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_all_rank", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.2, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "bert4rec_sasrec", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_config_in_batch.json b/configs/train/bert4rec_train_config_in_batch.json new file mode 100644 index 00000000..8a83211b --- /dev/null +++ b/configs/train/bert4rec_train_config_in_batch.json @@ -0,0 +1,169 @@ +{ + "experiment_name": "bert4rec_in_batch_beauty_bce_0-3_0-5", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.3, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_in_batch", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.5, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "bert4rec_sasrec", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_config_popular.json b/configs/train/bert4rec_train_config_popular.json new file mode 100644 index 00000000..2a05db02 --- /dev/null +++ b/configs/train/bert4rec_train_config_popular.json @@ -0,0 +1,170 @@ +{ + "experiment_name": "bert4rec_popular_beauty_bce_0-3_0-4", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.3, + "num_negatives_train": 100, + "type": "masked_item_prediction", + "negative_sampler_type": "random_by_popularity" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_popular", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "bert4rec_sasrec", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_config_random.json b/configs/train/bert4rec_train_config_random.json new file mode 100644 index 00000000..a72536f1 --- /dev/null +++ b/configs/train/bert4rec_train_config_random.json @@ -0,0 +1,170 @@ +{ + "experiment_name": "bert4rec_random_beauty_bce_0-5_0-4", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "mask_prob": 0.5, + "num_negatives_train": 100, + "type": "masked_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "bert4rec_popular", + "user_prefix": "user", + "sequence_prefix": "item", + "labels_prefix": "labels", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "bert4rec_sasrec", + "predictions_prefix": "logits", + "labels_prefix": "labels", + "output_prefix": "downstream_loss", + "weight": 1.0 + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/bert4rec_train_grid_config.json b/configs/train/bert4rec_train_grid_config.json index f9cb31cc..dedeb280 100644 --- a/configs/train/bert4rec_train_grid_config.json +++ b/configs/train/bert4rec_train_grid_config.json @@ -1,26 +1,26 @@ { "start_from": 0, - "experiment_name": "bert4rec_beauty_grid", + "experiment_name": "bert4rec_popular_beauty_0-4_0-2_num_neg", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { "type": "masked_item_prediction", - "negative_sampler_type": "random" + "negative_sampler_type": "random_by_popularity" } }, "dataset_params": { "samplers": { + "num_negatives_train": [ + 500, + 1000, + 5000 + ], "mask_prob": [ - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8 + 0.4 ] } }, @@ -45,7 +45,7 @@ } }, "model": { - "type": "bert4rec", + "type": "bert4rec_popular", "sequence_prefix": "item", "labels_prefix": "labels", "candidate_prefix": "candidates", @@ -59,14 +59,8 @@ }, "model_params": { "dropout": [ - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8] + 0.2 + ] }, "optimizer": { "type": "basic", diff --git a/configs/train/sasrec_train_config.json b/configs/train/sasrec_train_config.json index aa29a029..ad8dc2c5 100644 --- a/configs/train/sasrec_train_config.json +++ b/configs/train/sasrec_train_config.json @@ -3,7 +3,7 @@ "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { diff --git a/configs/train/sasrec_train_config_all_rank.json b/configs/train/sasrec_train_config_all_rank.json new file mode 100644 index 00000000..2db3f8ac --- /dev/null +++ b/configs/train/sasrec_train_config_all_rank.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_all_rank_beauty_loss_ce_0-7", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_all_rank", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.7, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_in_batch.json b/configs/train/sasrec_train_config_in_batch.json new file mode 100644 index 00000000..9b7f753f --- /dev/null +++ b/configs/train/sasrec_train_config_in_batch.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_in_batch_beauty_loss_ce_0-4", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_in_batch", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_in_batch_adaptive.json b/configs/train/sasrec_train_config_in_batch_adaptive.json new file mode 100644 index 00000000..5028390f --- /dev/null +++ b/configs/train/sasrec_train_config_in_batch_adaptive.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_in_batch_beauty_loss_ce_adaptive", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_in_batch", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce_adaptive", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_in_batch_log_q.json b/configs/train/sasrec_train_config_in_batch_log_q.json new file mode 100644 index 00000000..4c74030e --- /dev/null +++ b/configs/train/sasrec_train_config_in_batch_log_q.json @@ -0,0 +1,167 @@ +{ + "experiment_name": "sasrec_in_batch_beauty_loss_ce_log_q", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_in_batch", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.4, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce_log_q", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_popular.json b/configs/train/sasrec_train_config_popular.json new file mode 100644 index 00000000..f968ff7c --- /dev/null +++ b/configs/train/sasrec_train_config_popular.json @@ -0,0 +1,168 @@ +{ + "experiment_name": "sasrec_popular_beauty_loss_ce_0-6", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "num_negatives_train": 100, + "type": "next_item_prediction", + "negative_sampler_type": "random_by_popularity" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_popular", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.6, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_config_random.json b/configs/train/sasrec_train_config_random.json new file mode 100644 index 00000000..890bbb24 --- /dev/null +++ b/configs/train/sasrec_train_config_random.json @@ -0,0 +1,168 @@ +{ + "experiment_name": "sasrec_random_beauty_loss_ce_0-5", + "best_metric": "validation/ndcg@20", + "dataset": { + "type": "sequence", + "path_to_data_dir": "../data/sasrec_in_batch", + "name": "Beauty", + "max_sequence_length": 50, + "samplers": { + "num_negatives_train": 100, + "type": "next_item_prediction", + "negative_sampler_type": "random" + } + }, + "dataloader": { + "train": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": true, + "shuffle": true + }, + "validation": { + "type": "torch", + "batch_size": 256, + "batch_processor": { + "type": "basic" + }, + "drop_last": false, + "shuffle": false + } + }, + "model": { + "type": "sasrec_popular", + "sequence_prefix": "item", + "positive_prefix": "positive", + "negative_prefix": "negative", + "candidate_prefix": "candidates", + "embedding_dim": 64, + "num_heads": 2, + "num_layers": 2, + "dim_feedforward": 256, + "dropout": 0.5, + "activation": "gelu", + "layer_norm_eps": 1e-9, + "initializer_range": 0.02 + }, + "optimizer": { + "type": "basic", + "optimizer": { + "type": "adam", + "lr": 0.001 + }, + "clip_grad_threshold": 5.0 + }, + "loss": { + "type": "composite", + "losses": [ + { + "type": "sasrec_ce", + "positive_prefix": "positive_scores", + "negative_prefix": "negative_scores", + "output_prefix": "downstream_loss" + } + ], + "output_prefix": "loss" + }, + "callback": { + "type": "composite", + "callbacks": [ + { + "type": "metric", + "on_step": 1, + "loss_prefix": "loss" + }, + { + "type": "validation", + "on_step": 64, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + }, + { + "type": "eval", + "on_step": 256, + "pred_prefix": "logits", + "labels_prefix": "labels", + "metrics": { + "ndcg@5": { + "type": "ndcg", + "k": 5 + }, + "ndcg@10": { + "type": "ndcg", + "k": 10 + }, + "ndcg@20": { + "type": "ndcg", + "k": 20 + }, + "recall@5": { + "type": "recall", + "k": 5 + }, + "recall@10": { + "type": "recall", + "k": 10 + }, + "recall@20": { + "type": "recall", + "k": 20 + }, + "coverage@5": { + "type": "coverage", + "k": 5 + }, + "coverage@10": { + "type": "coverage", + "k": 10 + }, + "coverage@20": { + "type": "coverage", + "k": 20 + } + } + } + ] + } +} diff --git a/configs/train/sasrec_train_grid_config.json b/configs/train/sasrec_train_grid_config.json index 2eb99b8a..7ccb7fc6 100644 --- a/configs/train/sasrec_train_grid_config.json +++ b/configs/train/sasrec_train_grid_config.json @@ -1,10 +1,10 @@ { "start_from": 0, - "experiment_name": "sasrec_beauty_grid", + "experiment_name": "sasrec_grid_random_beauty_num_neg", "best_metric": "validation/ndcg@20", "dataset": { "type": "sequence", - "path_to_data_dir": "../data", + "path_to_data_dir": "../data/sasrec_in_batch", "name": "Beauty", "max_sequence_length": 50, "samplers": { @@ -13,6 +13,11 @@ } }, "dataset_params": { + "num_negatives_train": [ + 100, + 500, + 1000 + ] }, "dataloader": { "train": { @@ -35,7 +40,7 @@ } }, "model": { - "type": "sasrec", + "type": "sasrec_popular", "sequence_prefix": "item", "positive_prefix": "positive", "negative_prefix": "negative", @@ -50,15 +55,8 @@ }, "model_params": { "dropout": [ - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9] + 0.5 + ] }, "optimizer": { "type": "basic", @@ -74,7 +72,7 @@ "type": "composite", "losses": [ { - "type": "sasrec", + "type": "sasrec_ce", "positive_prefix": "positive_scores", "negative_prefix": "negative_scores", "output_prefix": "downstream_loss" diff --git a/modeling/dataset/negative_samplers/__init__.py b/modeling/dataset/negative_samplers/__init__.py index 498de21f..70db4559 100644 --- a/modeling/dataset/negative_samplers/__init__.py +++ b/modeling/dataset/negative_samplers/__init__.py @@ -1,9 +1,11 @@ from .base import BaseNegativeSampler from .popular import PopularNegativeSampler from .random import RandomNegativeSampler +from .random_by_popularity import RandomByPopularityNegativeSampler __all__ = [ 'BaseNegativeSampler', 'PopularNegativeSampler', - 'RandomNegativeSampler' + 'RandomNegativeSampler', + 'RandomByPopularityNegativeSampler' ] diff --git a/modeling/dataset/negative_samplers/random_by_popularity.py b/modeling/dataset/negative_samplers/random_by_popularity.py new file mode 100644 index 00000000..4d209dfa --- /dev/null +++ b/modeling/dataset/negative_samplers/random_by_popularity.py @@ -0,0 +1,59 @@ +from dataset.negative_samplers.base import BaseNegativeSampler + +from collections import Counter +import torch + + +class RandomByPopularityNegativeSampler(BaseNegativeSampler, config_name='random_by_popularity'): + + def __init__( + self, + dataset, + num_users, + num_items + ): + super().__init__( + dataset=dataset, + num_users=num_users, + num_items=num_items + ) + + self._item_popularity = self._compute_item_popularity() + + @classmethod + def create_from_config(cls, _, **kwargs): + return cls( + dataset=kwargs['dataset'], + num_users=kwargs['num_users'], + num_items=kwargs['num_items'] + ) + + def _compute_item_popularity(self): + popularity = Counter() + + for sample in self._dataset: + for item_id in sample['item.ids']: + popularity[item_id] += 1 + + # Convert to tensor for efficient sampling + items = list(popularity.keys()) + weights = torch.tensor(list(popularity.values()), dtype=torch.float32) + return {"items": items, "weights": weights} + + def generate_negative_samples(self, sample, num_negatives): + user_id = sample['user.ids'][0] + seen_items = set(self._seen_items[user_id]) # Convert to set for faster lookup + + items = self._item_popularity["items"] + weights = self._item_popularity["weights"] + + negatives = [] + while len(negatives) < num_negatives: + sampled_indices = torch.multinomial(weights, num_samples=num_negatives, replacement=False) + sampled_items = [items[idx] for idx in sampled_indices] + + for item in sampled_items: + if item not in seen_items and len(negatives) < num_negatives: + negatives.append(item) + + return negatives diff --git a/modeling/dataset/samplers/masked_item_prediction.py b/modeling/dataset/samplers/masked_item_prediction.py index 0100f731..f9170f19 100644 --- a/modeling/dataset/samplers/masked_item_prediction.py +++ b/modeling/dataset/samplers/masked_item_prediction.py @@ -1,4 +1,5 @@ from dataset.samplers.base import TrainSampler, EvalSampler +from dataset.negative_samplers.base import BaseNegativeSampler import copy import numpy as np @@ -6,21 +7,27 @@ class MaskedItemPredictionTrainSampler(TrainSampler, config_name='masked_item_prediction'): - def __init__(self, dataset, num_users, num_items, mask_prob=0.0): + def __init__(self, dataset, num_users, num_items, negative_sampler, mask_prob=0.0, num_negatives=0): super().__init__() self._dataset = dataset self._num_users = num_users self._num_items = num_items self._mask_item_idx = self._num_items + 1 self._mask_prob = mask_prob + self._negative_sampler = negative_sampler + self._num_negatives = num_negatives @classmethod def create_from_config(cls, config, **kwargs): + negative_sampler = BaseNegativeSampler.create_from_config({'type': config['negative_sampler_type']}, **kwargs) + return cls( dataset=kwargs['dataset'], num_users=kwargs['num_users'], num_items=kwargs['num_items'], - mask_prob=config.get('mask_prob', 0.0) + negative_sampler=negative_sampler, + mask_prob=config.get('mask_prob', 0.0), + num_negatives=config.get('num_negatives_train', 0) ) def __getitem__(self, index): @@ -53,17 +60,38 @@ def __getitem__(self, index): masked_sequence[-1] = self._mask_item_idx labels[-1] = item_sequence[-1] - return { - 'user.ids': sample['user.ids'], - 'user.length': sample['user.length'], + if self._num_negatives == 0: + return { + 'user.ids': sample['user.ids'], + 'user.length': sample['user.length'], - 'item.ids': masked_sequence, - 'item.length': len(masked_sequence), + 'item.ids': masked_sequence, + 'item.length': len(masked_sequence), - 'labels.ids': labels, - 'labels.length': len(labels) - } + 'labels.ids': labels, + 'labels.length': len(labels), + + 'not_masked_item.ids': item_sequence, + 'not_masked_item.length': len(item_sequence) + } + else: + negative_sequence = self._negative_sampler.generate_negative_samples( + sample, self._num_negatives + ) + + return { + 'user.ids': sample['user.ids'], + 'user.length': sample['user.length'], + + 'item.ids': masked_sequence, + 'item.length': len(masked_sequence), + + 'labels.ids': labels, + 'labels.length': len(labels), + 'negative_item.ids': negative_sequence, + 'negative_item.length': len(negative_sequence) + } class MaskedItemPredictionEvalSampler(EvalSampler, config_name='masked_item_prediction'): diff --git a/modeling/dataset/samplers/next_item_prediction.py b/modeling/dataset/samplers/next_item_prediction.py index c141b065..c4290b0c 100644 --- a/modeling/dataset/samplers/next_item_prediction.py +++ b/modeling/dataset/samplers/next_item_prediction.py @@ -1,7 +1,37 @@ from dataset.samplers.base import TrainSampler, EvalSampler from dataset.negative_samplers.base import BaseNegativeSampler +import torch import copy +from collections import defaultdict +import numpy as np + + +class DatasetAnalyzer: + def __init__(self, dataset): + self.dataset = dataset + self.item_freq = None # Словарь для хранения частот + self.all_items_count = 0 + + def precompute_frequencies(self): + """Подсчет частот встречаемости item_id в dataset""" + freq = defaultdict(int) + + # Проходим по всем элементам dataset + for sample in self.dataset: + for item_id in sample['item.ids']: + freq[item_id] += 1 + self.all_items_count += len(sample['item.ids']) + + self.item_freq = freq + + def get_frequency(self, item_ids): + """Возвращает список частот для каждого item_id из списка item_ids""" + if not isinstance(item_ids, list): + raise TypeError("item_ids должен быть списком") + + # Для каждого item_id в списке возвращаем его частоту или 0, если его нет + return [self.item_freq.get(item_id, 0) for item_id in item_ids] class NextItemPredictionTrainSampler(TrainSampler, config_name='next_item_prediction'): @@ -13,6 +43,8 @@ def __init__(self, dataset, num_users, num_items, negative_sampler, num_negative self._num_items = num_items self._negative_sampler = negative_sampler self._num_negatives = num_negatives + self._frequency_counter = DatasetAnalyzer(dataset) + self._frequency_counter.precompute_frequencies() @classmethod def create_from_config(cls, config, **kwargs): @@ -26,6 +58,7 @@ def create_from_config(cls, config, **kwargs): num_negatives=config.get('num_negatives_train', 0) ) + # here add how often each element in batch is occur def __getitem__(self, index): sample = copy.deepcopy(self._dataset[index]) @@ -40,8 +73,17 @@ def __getitem__(self, index): 'item.ids': item_sequence, 'item.length': len(item_sequence), + 'item_counts.ids': self._frequency_counter.get_frequency(item_sequence), + 'item_counts.length': len(item_sequence), + 'positive.ids': next_item_sequence, - 'positive.length': len(next_item_sequence) + 'positive.length': len(next_item_sequence), + + 'positive_counts.ids': self._frequency_counter.get_frequency(next_item_sequence), + 'positive_counts.length': len(next_item_sequence), + + 'counts.ids': [self._frequency_counter.all_items_count], + 'counts.length': 1, } else: negative_sequence = self._negative_sampler.generate_negative_samples( diff --git a/modeling/loss/base.py b/modeling/loss/base.py index dc03d6cb..6884d309 100644 --- a/modeling/loss/base.py +++ b/modeling/loss/base.py @@ -106,6 +106,135 @@ def forward(self, inputs): return loss +class CrossEntropyLossSasrec(TorchLoss, config_name='sasrec_ce'): + + def __init__( + self, + positive_prefix, + negative_prefix, + output_prefix=None + ): + super().__init__() + self._positive_prefix = positive_prefix + self._negative_prefix = negative_prefix + self._output_prefix = output_prefix + + self._loss = nn.CrossEntropyLoss() + + def forward(self, inputs): + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) + assert positive_scores.shape[0] == negative_scores.shape[0] + + all_logits = torch.cat((positive_scores, negative_scores), dim=1) + all_labels = torch.zeros(len(all_logits), dtype=torch.long, device=all_logits.device) + assert all_logits.shape[0] == all_labels.shape[0] + + loss = self._loss(all_logits, all_labels) # (1) + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss + +# positive_logits (all_samples) +# negative_logits (all_samples, num_negatives) +# all_counts () +# negative_counts (num_negatives) +def process_log_q(positive_logits, negative_logits, all_counts, negative_counts): + log_q = torch.log(negative_counts / all_counts) # (num_negatives) + adjusted_neg_logits = negative_logits - log_q # (all_samples, num_negatives) + + combined_logits = torch.cat([positive_logits.unsqueeze(1), adjusted_neg_logits], dim=1) # (all_samples, num_negatives + 1) + + loss = -torch.logsoftmax(combined_logits, dim=-1)[:, 0].mean() + + return loss + +class CrossEntropyLossSasrecLogQ(TorchLoss, config_name='sasrec_ce_log_q'): + + def __init__( + self, + positive_prefix, + negative_prefix, + output_prefix=None + ): + super().__init__() + self._positive_prefix = positive_prefix + self._negative_prefix = negative_prefix + self._output_prefix = output_prefix + + def forward(self, inputs): + # import code + # code.interact(local=locals()) + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) + assert positive_scores.shape[0] == negative_scores.shape[0] + all_counts = inputs['all_counts'] # () + negative_counts = inputs['negative_counts'] # (num_negatives) + + log_q = torch.log(negative_counts / all_counts) # (num_negatives) + negative_scores = negative_scores - log_q # (all_samples, num_negatives) + + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (all_samples, num_negatives + 1) + loss = -torch.log_softmax(all_logits, dim=-1)[:, 0].mean() # (1) + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss + +def log_softmax_without_self(logits: torch.Tensor, dim: int = -1) -> torch.Tensor: + """ + Вычисляет log-softmax, исключая текущий элемент из знаменателя. + """ + log_softmax_normal = torch.log_softmax(logits, dim=dim) + correction = torch.log(-torch.expm1(log_softmax_normal) + 1e-8) + return log_softmax_normal - correction + +class CrossEntropyLossSasrecAdaptive(TorchLoss, config_name='sasrec_ce_adaptive'): + + def __init__( + self, + positive_prefix, + negative_prefix, + output_prefix=None + ): + super().__init__() + self._positive_prefix = positive_prefix + self._negative_prefix = negative_prefix + self._output_prefix = output_prefix + + def forward(self, inputs): + # import code + # code.interact(local=locals()) + positive_scores = inputs[self._positive_prefix].unsqueeze(1) # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) + assert positive_scores.shape[0] == negative_scores.shape[0] + all_counts = inputs['all_counts'] # () + negative_counts = inputs['negative_counts'] # (num_negatives) + positive_counts = inputs['positive_counts'] # (x) + + batch_size = negative_scores.shape[-1] + + log_q = torch.log(negative_counts[None, :] / (all_counts - positive_counts[:, None])) # (x, num_negatives) + negative_scores = negative_scores - log_q # (x, num_negatives) + + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (all_samples, num_negatives + 1) + loss = -torch.log_softmax(all_logits, dim=-1)[:, 0] # (x) + + negative_scores = negative_scores - log_q - torch.log(torch.ones_like(negative_scores) * (batch_size - 1)) # (x, num_negatives) + all_logits = torch.cat((positive_scores, negative_scores), dim=1) # (all_samples, num_negatives + 1) + prob = torch.softmax(all_logits, dim=-1)[:, 0] # (x) + multiplier = (1.0 - prob).detach() # (x) + + loss *= multiplier + loss = loss.mean() + + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss + + class BinaryCrossEntropyLoss(TorchLoss, config_name='bce'): def __init__( @@ -255,12 +384,12 @@ def __init__( self._output_prefix = output_prefix def forward(self, inputs): - positive_scores = inputs[self._positive_prefix] # (x, embedding_dim) - negative_scores = inputs[self._negative_prefix] # (x, embedding_dim) + positive_scores = inputs[self._positive_prefix] # (x, 1) + negative_scores = inputs[self._negative_prefix] # (x, num_negatives) assert positive_scores.shape[0] == negative_scores.shape[0] - positive_loss = torch.log(nn.functional.sigmoid(positive_scores)).sum(dim=-1) # (x) - negative_loss = torch.log(1.0 - nn.functional.sigmoid(negative_scores)).sum(dim=-1) # (x) + positive_loss = torch.log(nn.functional.sigmoid(positive_scores) + 1e-9).sum(dim=-1) # (x) + negative_loss = torch.log(1.0 - nn.functional.sigmoid(negative_scores) + 1e-9).sum(dim=-1) # (x) loss = positive_loss + negative_loss # (x) loss = -loss.sum() # (1) @@ -269,6 +398,36 @@ def forward(self, inputs): return loss +class Bert4recSASRecLoss(TorchLoss, config_name='bert4rec_sasrec'): + + def __init__(self, predictions_prefix, labels_prefix, output_prefix=None): + super().__init__() + self._pred_prefix = predictions_prefix + self._labels_prefix = labels_prefix + self._output_prefix = output_prefix + + def forward(self, inputs): + all_logits = inputs[self._pred_prefix] # (all_items, num_classes) + all_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_items) + assert all_logits.shape[0] == all_labels.shape[0] + + # Extract positive scores using gather + positive_scores = all_logits.gather(1, all_labels.unsqueeze(1)).squeeze(1) + # Create a mask to exclude the positive indices + mask = torch.arange(all_logits.size(1), device=all_logits.device) != all_labels.unsqueeze(1) + # Apply the mask and reshape to get negative scores + negative_scores = all_logits[mask].view(all_logits.size(0), -1) + assert positive_scores.shape[0] == negative_scores.shape[0] + + positive_loss = torch.log(nn.functional.sigmoid(positive_scores) + 1e-9).sum(dim=-1) # (x) + negative_loss = torch.log(1.0 - nn.functional.sigmoid(negative_scores) + 1e-9).sum(dim=-1) # (x) + loss = positive_loss + negative_loss # (x) + loss = -loss.sum() # (1) + + if self._output_prefix is not None: + inputs[self._output_prefix] = loss.cpu().item() + + return loss class SamplesSoftmaxLoss(TorchLoss, config_name='sampled_softmax'): diff --git a/modeling/models/__init__.py b/modeling/models/__init__.py index b4341e75..80d60b44 100644 --- a/modeling/models/__init__.py +++ b/modeling/models/__init__.py @@ -1,6 +1,9 @@ from .base import BaseModel, SequentialTorchModel, TorchModel from .bert4rec import Bert4RecModel from .bert4rec_cls import Bert4RecModelCLS +from .bert4rec_all_rank import Bert4RecModelAllRank +from .bert4rec_in_batch import Bert4RecModelInBatch +from .bert4rec_popular import Bert4RecModelPopular from .cl4srec import Cl4SRecModel from .duorec import DuoRecModel from .graph_seq_rec import GraphSeqRecModel @@ -12,6 +15,6 @@ from .pop import PopModel from .pure_mf import PureMF from .random import RandomModel -from .sasrec import SasRecModel, SasRecInBatchModel +from .sasrec import SasRecModel, SasRecModelAllRank, SasRecModelPopular, SasRecInBatchModel from .sasrec_ce import SasRecCeModel from .s3rec import S3RecModel diff --git a/modeling/models/bert4rec.py b/modeling/models/bert4rec.py index 40f1d331..baf4f774 100644 --- a/modeling/models/bert4rec.py +++ b/modeling/models/bert4rec.py @@ -72,11 +72,11 @@ def forward(self, inputs): ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) - embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) embeddings = torch.einsum( 'bsd,nd->bsn', embeddings, self._item_embeddings.weight ) # (batch_size, seq_len, num_items) - embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) if self.training: # training mode all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) diff --git a/modeling/models/bert4rec_all_rank.py b/modeling/models/bert4rec_all_rank.py new file mode 100644 index 00000000..d9404953 --- /dev/null +++ b/modeling/models/bert4rec_all_rank.py @@ -0,0 +1,100 @@ +from models.base import SequentialTorchModel + +import torch +import torch.nn as nn + + +class Bert4RecModelAllRank(SequentialTorchModel, config_name='bert4rec_all_rank'): + + def __init__( + self, + sequence_prefix, + labels_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='gelu', + layer_norm_eps=1e-5, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=False + ) + self._sequence_prefix = sequence_prefix + self._labels_prefix = labels_prefix + + self._output_projection = nn.Linear( + in_features=embedding_dim, + out_features=embedding_dim + ) + + self._bias = nn.Parameter( + data=torch.zeros(num_items + 2), + requires_grad=True + ) + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + labels_prefix=config['labels_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + embeddings = torch.einsum( + 'bsd,nd->bsn', embeddings, self._item_embeddings.weight + ) # (batch_size, seq_len, num_items) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + + if self.training: # training mode + all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) + embeddings = embeddings[mask] # (all_batch_events, num_items) + labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + + needed_logits = embeddings[labels_mask] # (non_zero_events, num_items) + needed_labels = all_sample_labels[labels_mask] # (non_zero_events) + + return {'logits': needed_logits, 'labels.ids': needed_labels} + else: # eval mode + candidate_scores = self._get_last_embedding(embeddings, mask) # (batch_size, num_items) + candidate_scores[:, 0] = -torch.inf + candidate_scores[:, self._num_items + 1:] = -torch.inf + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices diff --git a/modeling/models/bert4rec_in_batch.py b/modeling/models/bert4rec_in_batch.py new file mode 100644 index 00000000..27671843 --- /dev/null +++ b/modeling/models/bert4rec_in_batch.py @@ -0,0 +1,121 @@ +from models.base import SequentialTorchModel + +import torch +import torch.nn as nn + + +class Bert4RecModelInBatch(SequentialTorchModel, config_name='bert4rec_in_batch'): + + def __init__( + self, + sequence_prefix, + labels_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='gelu', + layer_norm_eps=1e-5, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=False + ) + self._sequence_prefix = sequence_prefix + self._labels_prefix = labels_prefix + + self._output_projection = nn.Linear( + in_features=embedding_dim, + out_features=embedding_dim + ) + + self._bias = nn.Parameter( + data=torch.zeros(num_items + 2), + requires_grad=True + ) + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + labels_prefix=config['labels_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings = torch.nn.functional.gelu(embeddings) # (batch_size, seq_len, embedding_dim) + # embeddings += self._bias[None, None, :] # (batch_size, seq_len, num_items) + + if self.training: # training mode + # TODO: move 'not_masked_item' to config + all_sample_not_masked = inputs['not_masked_item.ids'] # (all_batch_events) + + random_indices = torch.randperm(all_sample_not_masked.shape[0])[:embeddings.shape[0]] # (batch_size) + random_in_batch_negative_ids = all_sample_not_masked[random_indices] # (batch_size) + random_in_batch_negative_embeddings = self._item_embeddings.weight[random_in_batch_negative_ids] # (batch_size) + + embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) + labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + non_zero_embeddings = embeddings[labels_mask] # (non_zero_events, embedding_dim) + non_zero_labels = all_sample_labels[labels_mask] # (non_zero_events) + + # non_zero_samples_logits = torch.einsum( + # 'bd,nd->bn', non_zero_embeddings, random_in_batch_negative_embeddings + # ) # (non_zero_events, num_negatives=batch_size) + non_zero_samples_logits = non_zero_embeddings @ random_in_batch_negative_embeddings.T # (non_zero_events, num_negatives=batch_size) + non_zero_labels_embeddings = self._item_embeddings.weight[non_zero_labels] # (non_zero_events, embedding_dim) + # non_zero_labels_logits = (non_zero_embeddings @ non_zero_labels_embeddings.T).diagonal().unsqueeze(1) # (non_zero_events, 1) + non_zero_labels_logits = (non_zero_embeddings * non_zero_labels_embeddings).sum(dim=-1).unsqueeze(1) # (non_zero_events, 1) + + needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1=batch_size + 1) + # needed_labels = all_sample_labels[labels_mask] # (non_zero_events) + + needed_labels = torch.zeros(len(needed_logits), dtype=torch.long, device=needed_logits.device) # (non_zero_events) + + return {'logits': needed_logits, 'labels.ids': needed_labels} + else: + # eval mode + embeddings = torch.einsum( + 'bsd,nd->bsn', embeddings, self._item_embeddings.weight + ) # (batch_size, seq_len, num_items) + + candidate_scores = self._get_last_embedding(embeddings, mask) # (batch_size, num_items) + candidate_scores[:, 0] = -torch.inf + candidate_scores[:, self._num_items + 1:] = -torch.inf + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices diff --git a/modeling/models/bert4rec_popular.py b/modeling/models/bert4rec_popular.py new file mode 100644 index 00000000..18c403b9 --- /dev/null +++ b/modeling/models/bert4rec_popular.py @@ -0,0 +1,115 @@ +from models.base import SequentialTorchModel + +import torch +import torch.nn as nn + + +class Bert4RecModelPopular(SequentialTorchModel, config_name='bert4rec_popular'): + + def __init__( + self, + sequence_prefix, + labels_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='gelu', + layer_norm_eps=1e-5, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=False + ) + self._sequence_prefix = sequence_prefix + self._labels_prefix = labels_prefix + + self._output_projection = nn.Linear( + in_features=embedding_dim, + out_features=embedding_dim + ) + + self._bias = nn.Parameter( + data=torch.zeros(num_items + 2), + requires_grad=True + ) + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + labels_prefix=config['labels_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + embeddings = self._output_projection(embeddings) # (batch_size, seq_len, embedding_dim) + + if self.training: # training mode + # TODO: move 'negative_item' to config + negative_items = inputs['negative_item.ids'] # (num_negatives) + + negative_embeddings = self._item_embeddings.weight[negative_items] # (batch_size * num_negatives, embedding_dim) + negative_embeddings = negative_embeddings.reshape(len(all_sample_lengths), int(negative_embeddings.size(0) / len(all_sample_lengths)), negative_embeddings.size(1)) # (batch_size, num_negatives, embedding_dim) + negative_embeddings = torch.repeat_interleave(negative_embeddings, all_sample_lengths, dim=0) # (all_batch_events, num_negatives, embedding_dim) + + embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + all_sample_labels = inputs['{}.ids'.format(self._labels_prefix)] # (all_batch_events) + labels_mask = (all_sample_labels != 0).bool() # (all_batch_events) + non_zero_negative_embeddings = negative_embeddings[labels_mask] # (non_zero_events, num_negatives, embedding_dim) + non_zero_embeddings = embeddings[labels_mask] # (non_zero_events, embedding_dim) + non_zero_labels = all_sample_labels[labels_mask] # (non_zero_events) + + non_zero_samples_logits = torch.einsum("bd,bnd->bn", non_zero_embeddings, non_zero_negative_embeddings) # (non_zero_events, num_negatives) + # non_zero_samples_logits = non_zero_embeddings @ non_zero_negative_embeddings.T # (non_zero_events, num_negatives) + non_zero_labels_embeddings = self._item_embeddings.weight[non_zero_labels] # (non_zero_events, embedding_dim) + non_zero_labels_logits = (non_zero_embeddings * non_zero_labels_embeddings).sum(dim=-1).unsqueeze(1) # (non_zero_events, 1) + + needed_logits = torch.cat((non_zero_labels_logits, non_zero_samples_logits), dim=1) # (non_zero_events, num_negatives + 1) + + needed_labels = torch.zeros(len(needed_logits), dtype=torch.long, device=needed_logits.device) # (non_zero_events) + + return {'logits': needed_logits, 'labels.ids': needed_labels} + else: # eval mode + embeddings = torch.einsum( + 'bsd,nd->bsn', embeddings, self._item_embeddings.weight + ) # (batch_size, seq_len, num_items) + + candidate_scores = self._get_last_embedding(embeddings, mask) # (batch_size, num_items) + candidate_scores[:, 0] = -torch.inf + candidate_scores[:, self._num_items + 1:] = -torch.inf + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices diff --git a/modeling/models/sasrec.py b/modeling/models/sasrec.py index 1ef5c9e7..a751910f 100644 --- a/modeling/models/sasrec.py +++ b/modeling/models/sasrec.py @@ -119,6 +119,218 @@ def forward(self, inputs): return indices +class SasRecModelAllRank(SequentialTorchModel, config_name='sasrec_all_rank'): + + def __init__( + self, + sequence_prefix, + positive_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='relu', + layer_norm_eps=1e-9, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=True + ) + self._sequence_prefix = sequence_prefix + self._positive_prefix = positive_prefix + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + positive_prefix=config['positive_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + if self.training: # training mode + all_positive_sample_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) + + all_sample_embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + + all_embeddings = self._item_embeddings.weight # (num_items + 2, embedding_dim) + + # a -- all_batch_events, n -- num_items + 2, d -- embedding_dim + all_scores = torch.einsum( + 'ad,nd->an', + all_sample_embeddings, + all_embeddings + ) # (all_batch_events, num_items + 2) + + positive_scores = torch.gather( + input=all_scores, + dim=1, + index=all_positive_sample_events[..., None] + ).squeeze() # (all_batch_items,) + + sample_ids, _ = create_masked_tensor( + data=all_sample_events, + lengths=all_sample_lengths + ) # (batch_size, seq_len) + + sample_ids = torch.repeat_interleave(sample_ids, all_sample_lengths, dim=0) # (all_batch_events, seq_len) + + negative_scores = torch.scatter( + input=all_scores, + dim=1, + index=sample_ids, + src=torch.ones_like(sample_ids) * (-torch.inf) + ) # (all_batch_events, num_items + 2) + negative_scores[:, 0] = -torch.inf # Padding idx + negative_scores[:, self._num_items + 1:] = -torch.inf # Mask idx + + return { + 'positive_scores': positive_scores, + 'negative_scores': negative_scores + } + else: # eval mode + last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) + # b - batch_size, n - num_candidates, d - embedding_dim + candidate_scores = torch.einsum( + 'bd,nd->bn', + last_embeddings, + self._item_embeddings.weight + ) # (batch_size, num_items + 2) + candidate_scores[:, 0] = -torch.inf # Padding id + candidate_scores[:, self._num_items + 1:] = -torch.inf # Mask id + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices + + +class SasRecModelPopular(SequentialTorchModel, config_name='sasrec_popular'): + + def __init__( + self, + sequence_prefix, + positive_prefix, + num_items, + max_sequence_length, + embedding_dim, + num_heads, + num_layers, + dim_feedforward, + dropout=0.0, + activation='relu', + layer_norm_eps=1e-9, + initializer_range=0.02 + ): + super().__init__( + num_items=num_items, + max_sequence_length=max_sequence_length, + embedding_dim=embedding_dim, + num_heads=num_heads, + num_layers=num_layers, + dim_feedforward=dim_feedforward, + dropout=dropout, + activation=activation, + layer_norm_eps=layer_norm_eps, + is_causal=True + ) + self._sequence_prefix = sequence_prefix + self._positive_prefix = positive_prefix + + self._init_weights(initializer_range) + + @classmethod + def create_from_config(cls, config, **kwargs): + return cls( + sequence_prefix=config['sequence_prefix'], + positive_prefix=config['positive_prefix'], + num_items=kwargs['num_items'], + max_sequence_length=kwargs['max_sequence_length'], + embedding_dim=config['embedding_dim'], + num_heads=config.get('num_heads', int(config['embedding_dim'] // 64)), + num_layers=config['num_layers'], + dim_feedforward=config.get('dim_feedforward', 4 * config['embedding_dim']), + dropout=config.get('dropout', 0.0), + initializer_range=config.get('initializer_range', 0.02) + ) + + def forward(self, inputs): + all_sample_events = inputs['{}.ids'.format(self._sequence_prefix)] # (all_batch_events) + all_sample_lengths = inputs['{}.length'.format(self._sequence_prefix)] # (batch_size) + + embeddings, mask = self._apply_sequential_encoder( + all_sample_events, all_sample_lengths + ) # (batch_size, seq_len, embedding_dim), (batch_size, seq_len) + + if self.training: # training mode + # TODO: move 'negative' to config + negative_items = inputs['negative.ids'] # (num_negatives) + negative_embeddings = self._item_embeddings.weight[negative_items] # (batch_size*num_negatives, embedding_dim) + negative_embeddings = negative_embeddings.reshape(len(all_sample_lengths), int(negative_embeddings.size(0) / len(all_sample_lengths)), negative_embeddings.size(1)) # (batch_size, num_negatives, embedding_dim) + negative_embeddings = torch.repeat_interleave(negative_embeddings, all_sample_lengths, dim=0) # (all_batch_events, num_negatives, embedding_dim) + + all_sample_embeddings = embeddings[mask] # (all_batch_events, embedding_dim) + negative_scores = torch.einsum("bd,bnd->bn", all_sample_embeddings, negative_embeddings) # (all_batch_events, num_negatives) + # negative_scores = all_sample_embeddings @ negative_embeddings.T + + all_positive_sample_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) + all_positive_sample_embeddings = self._item_embeddings.weight[ + all_positive_sample_events] # (all_batch_events, embedding_dim) + + positive_scores = (all_sample_embeddings * all_positive_sample_embeddings).sum(dim=-1) + + return { + 'positive_scores': positive_scores, + 'negative_scores': negative_scores + } + else: # eval mode + last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) + # b - batch_size, n - num_candidates, d - embedding_dim + candidate_scores = torch.einsum( + 'bd,nd->bn', + last_embeddings, + self._item_embeddings.weight + ) # (batch_size, num_items + 2) + candidate_scores[:, 0] = -torch.inf # Padding id + candidate_scores[:, self._num_items + 1:] = -torch.inf # Mask id + + _, indices = torch.topk( + candidate_scores, + k=20, dim=-1, largest=True + ) # (batch_size, 20) + + return indices + class SasRecInBatchModel(SasRecModel, config_name='sasrec_in_batch'): @@ -181,23 +393,29 @@ def forward(self, inputs): # positives in_batch_positive_events = inputs['{}.ids'.format(self._positive_prefix)] # (all_batch_events) - in_batch_positive_embeddings = self._item_embeddings( - in_batch_positive_events - ) # (all_batch_events, embedding_dim) + in_batch_positive_embeddings = self._item_embeddings(in_batch_positive_events) # (all_batch_events, embedding_dim) + in_batch_positive_counts = inputs['positive_counts.ids'] # negatives - batch_size = all_sample_lengths.shape[0] - random_ids = torch.randperm(in_batch_positive_events.shape[0]) - in_batch_negative_ids = in_batch_positive_events[random_ids][:batch_size] + batch_size = all_sample_lengths.shape[0] # scalar + random_ids = torch.randperm(in_batch_positive_events.shape[0]) # (batch_size) + in_batch_negative_ids = in_batch_positive_events[random_ids][:batch_size] # (batch_size) + in_batch_negative_counts = in_batch_positive_counts[random_ids][:batch_size] # (batch_size) + all_counts = inputs['counts.ids'][0] # scalar + + in_batch_negative_embeddings = self._item_embeddings(in_batch_negative_ids) # (batch_size, embedding_dim) - in_batch_negative_embeddings = self._item_embeddings( - in_batch_negative_ids - ) # (batch_size, embedding_dim) + in_batch_negative_scores = in_batch_queries_embeddings @ in_batch_negative_embeddings.T + in_batch_positive_scores = (in_batch_queries_embeddings * in_batch_positive_embeddings).sum(dim=-1) + # import code + # code.interact(local=locals()) return { - 'query_embeddings': in_batch_queries_embeddings, - 'positive_embeddings': in_batch_positive_embeddings, - 'negative_embeddings': in_batch_negative_embeddings + 'positive_scores': in_batch_positive_scores, # [i] + 'negative_scores': in_batch_negative_scores, # [i] + 'positive_counts': in_batch_positive_counts, + 'negative_counts': in_batch_negative_counts, # all + 'all_counts': all_counts # } else: # eval mode last_embeddings = self._get_last_embedding(embeddings, mask) # (batch_size, embedding_dim) diff --git a/modeling/train.py b/modeling/train.py index d865620d..ef95a664 100644 --- a/modeling/train.py +++ b/modeling/train.py @@ -1,12 +1,13 @@ import utils from utils import parse_args, create_logger, DEVICE, fix_random_seed -from callbacks import BaseCallback +from callbacks import BaseCallback, EvalCallback, ValidationCallback from dataset import BaseDataset from dataloader import BaseDataloader from loss import BaseLoss from models import BaseModel from optimizer import BaseOptimizer +from infer import inference import copy import json @@ -124,7 +125,7 @@ def main(): logger.debug('Everything is ready for training process!') # Train process - _ = train( + best_model_checkpoint = train( dataloader=train_dataloader, model=model, optimizer=optimizer, @@ -135,6 +136,27 @@ def main(): best_metric=config.get('best_metric') ) + eval_model = BaseModel.create_from_config(config['model'], **dataset.meta).to(DEVICE) + eval_model.load_state_dict(best_model_checkpoint) + + for cl in callback._callbacks: + if isinstance(cl, EvalCallback): + metrics = cl._metrics + pred_prefix = cl._pred_prefix + labels_prefix = cl._labels_prefix + break + else: + for cl in callback._callbacks: + if isinstance(cl, ValidationCallback): + metrics = cl._metrics + pred_prefix = cl._pred_prefix + labels_prefix = cl._labels_prefix + break + else: + assert False + + inference(eval_dataloader, eval_model, metrics, pred_prefix, labels_prefix) + logger.debug('Saving model...') checkpoint_path = '../checkpoints/{}_final_state.pth'.format(config['experiment_name']) torch.save(model.state_dict(), checkpoint_path)