Updated random seeding

Author: PierreMarza

src/thunder/benchmark.py

@@ -119,9 +119,6 @@ def run_benchmark(cfg: DictConfig, model_cls: Callable = None) -> None:
     from .utils.dice_loss import multiclass_dice_loss
     from .utils.utils import set_seed
 
-    # Setting the random seed
-    set_seed(UtilsConstants.DEFAULT_SEED.value)
-
     # Getting device
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -177,6 +174,9 @@ def run_benchmark(cfg: DictConfig, model_cls: Callable = None) -> None:
         shutil.rmtree(res_folder)
     os.makedirs(res_folder)
 
+    # Setting the random seed
+    set_seed(UtilsConstants.DEFAULT_SEED.value)
+
     if task_type in ["linear_probing", "segmentation"]:
         # Model checkpoints folder
         ckpt_folder = os.path.join(

src/thunder/utils/utils.py

@@ -192,6 +192,8 @@ def set_seed(seed: int) -> None:
         torch.cuda.manual_seed(seed)
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
+        torch.use_deterministic_algorithms(True)
+        os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
 
 
 def wb_mask(