Add ilamb_input_format option to batch processing (#484)

* add ilamb_input format option in batch_processing

* pre-commit fix

* add test

* remove changes in batch_tipmip_config.yml

Author: rhaegar325

docs/source/CMORise_ILAMB_workflow.md

@@ -147,6 +147,12 @@ cmip_version: CMIP7
 input_folder: "/g/data/p73/archive/CMIP7/ACCESS-ESM1-6/production/historical-02"
 output_folder: "YOUR_OUTPUT_PATH"
 
+# When true, after all variables finish the monitor automatically creates an
+# <output_folder>/ilamb_input/ directory of <variable_id>.nc symlinks pointing
+# at the CMORised files — ready to use as ILAMB model input (see
+# "Preparing ILAMB-Ready Files" below). Default: false.
+ilamb_input_format: true
+
 # File patterns (relative to input_folder)
 # All atmosphere/land variables share the same pattern
 file_patterns:
@@ -353,6 +359,16 @@ output_folder/
 
 ## Preparing ILAMB-Ready Files
 
+:::{tip}
+If you set `ilamb_input_format: true` in the batch configuration (see above), the
+monitor already builds the model-input symlinks for you: once all variables
+finish, an `<output_folder>/ilamb_input/` directory is created containing one
+`<variable_id>.nc` symlink per CMORised file. You can point ILAMB's `MODELS`
+entry straight at that directory and skip the manual `create_ilamb_model_symlinks`
+step below — you still need the `DATA` observational link (Step 1) to complete the
+ILAMB-ROOT layout.
+:::
+
 ILAMB requires a specific directory layout called **ILAMB-ROOT**:
 
 ```

src/access_moppy/batch_cmoriser.py

@@ -877,6 +877,22 @@ def finalize_monitor(
             f"Warning: failed to write batch coordination report: {e}", file=sys.stderr
         )
 
+    if config.get("ilamb_input_format"):
+        output_dir = Path(db_path).parent
+        ilamb_dir = output_dir / "ilamb_input"
+        try:
+            from access_moppy.utilities import create_ilamb_model_symlinks
+
+            links = create_ilamb_model_symlinks(
+                output_dir, ilamb_dir, drs_format="auto", overwrite=True
+            )
+            print(f"Created {len(links)} ILAMB input symlink(s) in: {ilamb_dir}")
+        except Exception as e:
+            print(
+                f"Warning: failed to create ILAMB input symlinks: {e}",
+                file=sys.stderr,
+            )
+
     sidecar = Path(db_path).parent / SIDECAR_FILENAME
     try:
         sidecar.unlink()

tests/unit/test_batch_cmoriser.py

@@ -1163,6 +1163,56 @@ def test_finalize_tolerates_missing_sidecar(self, temp_dir):
             # No sidecar to begin with — should not raise
             finalize_monitor(tracker, {"variables": []}, "historical", db_path)
 
+    @pytest.mark.unit
+    def test_finalize_creates_ilamb_symlinks_when_enabled(self, temp_dir):
+        """ilamb_input_format=True builds <output>/ilamb_input/<var>.nc symlinks."""
+        db_path = temp_dir / "test.db"
+        # Flat-DRS CMORised output living alongside the db in output_folder.
+        tas = temp_dir / "tas_Amon_ACCESS_historical.nc"
+        pr = temp_dir / "pr_Amon_ACCESS_historical.nc"
+        tas.write_text("x")
+        pr.write_text("x")
+
+        with TaskTracker(db_path) as tracker:
+            config = {"variables": ["Amon.tas"], "ilamb_input_format": True}
+            finalize_monitor(tracker, config, "historical", db_path)
+
+        ilamb_dir = temp_dir / "ilamb_input"
+        assert (ilamb_dir / "tas.nc").is_symlink()
+        assert (ilamb_dir / "pr.nc").is_symlink()
+        assert (ilamb_dir / "tas.nc").resolve() == tas.resolve()
+        assert (ilamb_dir / "pr.nc").resolve() == pr.resolve()
+
+    @pytest.mark.unit
+    def test_finalize_skips_ilamb_symlinks_when_disabled(self, temp_dir):
+        """No ilamb_input directory is created when the flag is unset."""
+        db_path = temp_dir / "test.db"
+        (temp_dir / "tas_Amon_ACCESS_historical.nc").write_text("x")
+
+        with TaskTracker(db_path) as tracker:
+            finalize_monitor(tracker, {"variables": []}, "historical", db_path)
+
+        assert not (temp_dir / "ilamb_input").exists()
+
+    @pytest.mark.unit
+    def test_finalize_survives_ilamb_symlink_failure(self, temp_dir):
+        """A symlink-build error is swallowed so finalize still completes."""
+        db_path = temp_dir / "test.db"
+        sidecar = temp_dir / SIDECAR_FILENAME
+        sidecar.write_text("12345.gadi-pbs\n2026-05-15T00:00:00\n")
+
+        with TaskTracker(db_path) as tracker:
+            config = {"variables": [], "ilamb_input_format": True}
+            with patch(
+                "access_moppy.utilities.create_ilamb_model_symlinks",
+                side_effect=ValueError("multiple files for variable"),
+            ):
+                # Must not raise despite the symlink builder blowing up.
+                finalize_monitor(tracker, config, "historical", db_path)
+
+        # Finalize still ran to completion (sidecar cleanup happened).
+        assert not sidecar.exists()
+
 
 class TestMonitorMain:
     """Unit tests for monitor_main entry point: env validation and submit flow."""