Fix incorrect argument name for online WOSAC logging. (#260)

* Fix incorrect argument name.

* Small revert.

* Incorporate PR feedback.

Author: daphne-cornelisse

pufferlib/config/ocean/drive.ini

@@ -104,7 +104,7 @@ eval_interval = 1000
 ; Path to dataset used for evaluation
 map_dir = "resources/drive/binaries/training"
 ; Evaluation will run on the first num_maps maps in the map_dir directory
-num_maps = 20
+wosac_num_maps = 20
 backend = PufferEnv
 ; WOSAC (Waymo Open Sim Agents Challenge) evaluation settings
 ; If True, enables evaluation on realism metrics each time we save a checkpoint

pufferlib/pufferl.py

@@ -522,12 +522,12 @@ def train(self):
                         print(f"Failed to export model weights: {e}")
 
         if self.config["eval"]["wosac_realism_eval"] and (
-            self.epoch % self.config["eval"]["eval_interval"] == 0 or done_training
+            (self.epoch - 1) % self.config["eval"]["eval_interval"] == 0 or done_training
         ):
             pufferlib.utils.run_wosac_eval_in_subprocess(self.config, self.logger, self.global_step)
 
         if self.config["eval"]["human_replay_eval"] and (
-            self.epoch % self.config["eval"]["eval_interval"] == 0 or done_training
+            (self.epoch - 1) % self.config["eval"]["eval_interval"] == 0 or done_training
         ):
             pufferlib.utils.run_human_replay_eval_in_subprocess(self.config, self.logger, self.global_step)
 
@@ -1040,7 +1040,7 @@ def eval(env_name, args=None, vecenv=None, policy=None):
     wosac_enabled = args["eval"]["wosac_realism_eval"]
     human_replay_enabled = args["eval"]["human_replay_eval"]
     args["env"]["map_dir"] = args["eval"]["map_dir"]
-    args["env"]["num_maps"] = args["eval"]["num_maps"]
+    args["env"]["num_maps"] = args["eval"]["wosac_num_maps"]
     args["env"]["use_all_maps"] = True
     dataset_name = args["env"]["map_dir"].split("/")[-1]
 

pufferlib/utils.py

@@ -91,12 +91,6 @@ def run_wosac_eval_in_subprocess(config, logger, global_step):
         model_dir = os.path.join(config["data_dir"], f"{config['env']}_{run_id}")
         model_files = glob.glob(os.path.join(model_dir, "model_*.pt"))
 
-        if not model_files:
-            print("No model files found for WOSAC evaluation")
-            return
-
-        latest_cpt = max(model_files, key=os.path.getctime)
-
         # Prepare evaluation command
         eval_config = config.get("eval", {})
         cmd = [
@@ -105,12 +99,10 @@ def run_wosac_eval_in_subprocess(config, logger, global_step):
             "pufferlib.pufferl",
             "eval",
             config["env"],
-            "--load-model-path",
-            latest_cpt,
             "--eval.wosac-realism-eval",
             "True",
-            "--eval.wosac-num-agents",
-            str(eval_config.get("wosac_num_agents", 256)),
+            "--eval.wosac-num-maps",
+            str(eval_config.get("wosac_num_maps", 256)),
             "--eval.wosac-init-mode",
             str(eval_config.get("wosac_init_mode", "create_all_valid")),
             "--eval.wosac-control-mode",
@@ -127,6 +119,12 @@ def run_wosac_eval_in_subprocess(config, logger, global_step):
             str(eval_config.get("wosac_aggregate_results", True)),
         ]
 
+        if not model_files:
+            print("No model files found for WOSAC evaluation. Running WOSAC with random policy.")
+        elif len(model_files) > 0:
+            latest_cpt = max(model_files, key=os.path.getctime)
+            cmd.extend(["--load-model-path", latest_cpt])
+
         # Run WOSAC evaluation in subprocess
         result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, cwd=os.getcwd())