Add default hyperparams also to model args, not only to main.py. When training without validation set, num_workers was not being set - fixed.

Author: gabikadlecova

main.py

@@ -50,8 +50,7 @@ def reload_checkpoint(path, device=None):
     parser.add_argument('--epochs', default=108, type=int, help='#epochs of training')
     parser.add_argument('--validation_size', default=10000, type=int, help="Size of the validation set to split off.")
     parser.add_argument('--num_workers', default=0, type=int, help="Number of parallel workers for the train dataset.")
-    parser.add_argument('--learning_rate', default=0.02, type=float, help='base learning rate')
-    parser.add_argument('--lr_decay_method', default='COSINE_BY_STEP', type=str, help='learning decay method')
+    parser.add_argument('--learning_rate', default=0.2, type=float, help='base learning rate')
     parser.add_argument('--optimizer', default='rmsprop', type=str, help='Optimizer (sgd, rmsprop or rmsprop_tf)')
     parser.add_argument('--rmsprop_eps', default=1.0, type=float, help='RMSProp eps parameter.')
     parser.add_argument('--momentum', default=0.9, type=float, help='momentum')

nasbench_pytorch/datasets/cifar10.py

@@ -109,7 +109,7 @@ def prepare_dataset(batch_size, test_batch_size=256, root='./data/', use_validat
                                                    worker_init_fn=worker_fn)
     else:
         train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True,
-                                                   worker_init_fn=worker_fn)
+                                                   num_workers=num_workers, worker_init_fn=worker_fn)
         valid_loader = None
 
     test_set = torchvision.datasets.CIFAR10(root=root, train=False, download=True, transform=test_transform)

nasbench_pytorch/trainer.py

@@ -37,10 +37,10 @@ def train(net, train_loader, loss=None, optimizer=None, scheduler=None, grad_cli
 
     if optimizer is not None and not isinstance(optimizer, str):
         pass
-    elif optimizer is None or optimizer.lower() == 'sgd':
+    elif optimizer is None or optimizer.lower() == 'rmsprop':
+        optimizer = torch.optim.RMSprop(net.parameters(), lr=0.2, momentum=0.9, weight_decay=1e-4, eps=1.0)
+    elif optimizer.lower() == 'sgd':
         optimizer = torch.optim.SGD(net.parameters(), lr=0.025, momentum=0.9, weight_decay=1e-4)
-    elif optimizer.lower() == 'rmsprop':
-        optimizer = torch.optim.RMSprop(net.parameters(), lr=0.2, momentum=0.9, weight_decay=1e-4)
     elif optimizer.lower() == 'adam':
         optimizer = torch.optim.Adam(net.parameters())