feat: Implement long context extension with yarn

Author: rrutmann

config_files/training/config_example_coca.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum.pbin
   intervals:
     training_log_interval_in_steps: 2

config_files/training/config_lorem_ipsum_long_fsdp1.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum_long.pbin
     test_dataset_path: ./data/lorem_ipsum.pbin
   intervals:

config_files/training/config_lorem_ipsum_long_fsdp1_warmstart.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum_long.pbin
     test_dataset_path: ./data/lorem_ipsum.pbin
   intervals:

config_files/training/config_lorem_ipsum_long_fsdp2.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum_long.pbin
     test_dataset_path: ./data/lorem_ipsum.pbin
   intervals:

config_files/training/config_lorem_ipsum_long_fsdp2_pp.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum_long.pbin
     test_dataset_path: ./data/lorem_ipsum.pbin
   intervals:

config_files/training/config_lorem_ipsum_long_fsdp2_pp_tp.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum_long.pbin
     test_dataset_path: ./data/lorem_ipsum.pbin
   intervals:

config_files/training/config_lorem_ipsum_long_fsdp2_warmstart.yaml

@@ -11,6 +11,7 @@ settings:
     world_size: ${cuda_env:WORLD_SIZE}
   paths:
     checkpoint_saving_path: data/checkpoints
+    experiments_root_path: ${modalities_env:experiments_root_path}
     train_dataset_path: ./data/lorem_ipsum_long.pbin
     test_dataset_path: ./data/lorem_ipsum.pbin
   intervals: