refactor: TinyMyo configuration and model for improved training efficiency and logging

Author: MatteoFasulo

config/experiment/TinyMyo_finetune.yaml

@@ -32,7 +32,6 @@ finetune_pretrained: True
 resume: False
 
 layerwise_lr_decay: 0.90
-scheduler_type: cosine
 
 pretrained_checkpoint_path: null
 pretrained_safetensors_path: null
@@ -52,17 +51,10 @@ defaults:
   - override /task: finetune_task_TinyMyo
   - override /criterion: finetune_criterion
 
-masking:
-  patch_size: [1, 20]
-  masking_ratio: 0.50
-  unmasked_loss_coeff: 0.1
-
-input_normalization:
-  normalize: False
-
 model:
+  n_layer: 8
   num_classes: 6
-  classification_type: "ml"
+  task: "classification"
 
 trainer:
   accelerator: gpu
@@ -87,19 +79,20 @@ callbacks:
 
 optimizer:
   optim: 'AdamW'
-  lr: 5e-4
-  betas: [0.9, 0.98]
-  weight_decay: 0.01
+  lr: 1e-4
+  betas: [0.9, 0.999]
+  weight_decay: 1e-2
 
 scheduler:
   trainer: ${trainer}
-  min_lr: 1e-5
-  warmup_lr_init: 1e-5
+  min_lr: 5e-6
+  warmup_lr_init: 5e-6
   warmup_epochs: 5
   total_training_opt_steps: ${max_epochs}
   t_in_epochs: True
 
 wandb:
   entity: "TinyMyo"
   project: "TinyMyo"
-  save_dir: ${env:LOG_DIR}
+  save_dir: ${env:LOG_DIR}
+  offline: True

config/experiment/TinyMyo_pretrain.yaml

@@ -22,8 +22,8 @@ tag: EMG_pretrain
 gpus: -1
 num_nodes: 1
 num_workers: 8
-batch_size: 128
-max_epochs: 50
+batch_size: 512
+max_epochs: 30
 
 final_validate: True
 final_test: False
@@ -49,22 +49,23 @@ masking:
 input_normalization:
   normalize: True
 
-scheduler:
-  trainer: ${trainer}
-  min_lr: 1e-6
-  warmup_lr_init: 1e-6
-  warmup_epochs: 10
-  total_training_opt_steps: ${max_epochs}
-  t_in_epochs: True
+model:
+  n_layer: 8
+  drop_path: 0.0 # Stochastic depth disabled for pretraining
+  num_classes: 0 # No classification head for pretraining
+  task: pretraining
+
+criterion:
+  loss_type: 'smooth_l1'
 
 trainer:
   accelerator: gpu
   num_nodes: ${num_nodes}
   devices: ${gpus}
   strategy: auto
+  precision: "bf16-mixed"
   max_epochs: ${max_epochs}
-  gradient_clip_val: 3
-  accumulate_grad_batches: 8
+  gradient_clip_val: 1
 
 model_checkpoint:
   save_last: True
@@ -73,11 +74,21 @@ model_checkpoint:
   save_top_k: 1
 
 optimizer:
-  optim: 'AdamW'
-  lr: 1e-4
+  lr: 5e-4
   betas: [0.9, 0.98]
-  weight_decay: 0.01
+  weight_decay: 1e-2
+
+scheduler:
+  trainer: ${trainer}
+  min_lr: 1e-6
+  warmup_lr_init: 1e-6
+  warmup_epochs: 3
+  total_training_opt_steps: ${max_epochs}
+  t_in_epochs: True
 
 wandb:
+  entity: "TinyMyo"
   project: "TinyMyo"
   save_dir: ${env:LOG_DIR}
+  run_name: "TinyMyo-Pretraining"
+  offline: True