Fix

Author: rbeucher

src/access_mopper/batch_cmoriser.py

@@ -139,12 +139,10 @@ def main():
     with config_path.open() as f:
         config_data = yaml.safe_load(f)
 
-    # Use a shared location for the database
-    script_dir = Path("cmor_job_scripts")
-    script_dir.mkdir(exist_ok=True)
-
-    # Put database in the script directory (which should be on a shared filesystem)
-    db_path = script_dir / "cmor_tasks.db"
+    # Put database in output directory on scratch filesystem (accessible from compute nodes)
+    output_dir = Path(config_data["output_folder"])
+    output_dir.mkdir(parents=True, exist_ok=True)
+    db_path = output_dir / "cmor_tasks.db"
     tracker = TaskTracker(db_path)
 
     # Pre-populate all tasks
@@ -160,7 +158,7 @@ def main():
     DASHBOARD_SCRIPT = files("access_mopper.dashboard").joinpath("cmor_dashboard.py")
     start_dashboard(str(DASHBOARD_SCRIPT), str(db_path))
 
-    # Create directory for job scripts
+    # Create directory for job scripts (local to login node is fine)
     script_dir = Path("cmor_job_scripts")
     script_dir.mkdir(exist_ok=True)
 
@@ -171,7 +169,7 @@ def main():
     print(f"Submitting {len(variables)} CMORisation jobs...")
 
     for variable in variables:
-        # Create job script
+        # Create job script - pass the scratch database path
         script_path = create_job_script(variable, config_data, str(db_path), script_dir)
         print(f"Created job script: {script_path}")