changes to logging (#588)

Author: tomsilver

scripts/submit.py

@@ -12,6 +12,9 @@ def _run() -> None:
     args = utils.parse_args()
     utils.update_config(args)
     job_name = f"{CFG.experiment_id}_{CFG.seed}"
+    os.makedirs(CFG.log_dir, exist_ok=True)
+    logfile_pattern = os.path.join(CFG.log_dir,
+                                   f"{utils.get_config_path_str()}__%j.log")
     argsstr = " ".join(sys.argv[1:])
     mystr = f"#!/bin/bash\npython src/main.py {argsstr}"
     temp_run_file = "temp_run_file.sh"
@@ -20,14 +23,15 @@ def _run() -> None:
         f.write(mystr)
     cmd = ("sbatch -p normal --time=99:00:00 --partition=xeon-p8 "
            f"--nodes=1 --exclusive --job-name={job_name} "
-           f"-o /tmp/%j_log.out {temp_run_file}")
+           f"-o {logfile_pattern} {temp_run_file}")
     print(f"Running command: {cmd}")
     output = subprocess.getoutput(cmd)
     if "command not found" in output:
         os.remove(temp_run_file)
         raise Exception("Are you logged into supercloud?")
     os.remove(temp_run_file)
-    logfile = os.path.join(CFG.log_dir, f"{utils.get_config_path_str()}.log")
+    job_id = output.split()[-1]
+    logfile = logfile_pattern.replace("%j", str(job_id))
     print(f"Started job, see log with:\ntail -n 10000 -F {logfile}")
 
 

src/main.py

@@ -64,15 +64,10 @@ def main() -> None:
     args = utils.parse_args()
     utils.update_config(args)
     str_args = " ".join(sys.argv)
-    # Log to both stdout and a logfile.
-    os.makedirs(CFG.log_dir, exist_ok=True)
-    logfile = os.path.join(CFG.log_dir, f"{utils.get_config_path_str()}.log")
-    logging.basicConfig(
-        level=CFG.loglevel,
-        format="%(message)s",
-        handlers=[logging.FileHandler(logfile),
-                  logging.StreamHandler()])
-    logging.info(f"Logging to {logfile}.")
+    # Log to stderr.
+    logging.basicConfig(level=CFG.loglevel,
+                        format="%(message)s",
+                        handlers=[logging.StreamHandler()])
     logging.info(f"Running command: python {str_args}")
     logging.info("Full config:")
     logging.info(CFG)