filter_real_stderr_lines in arkane adapter

Remove clutter from ARC.log

Author: alongd

arc/output.py

@@ -24,6 +24,7 @@
     AEC_SECTION_START, AEC_SECTION_END,
     MBAC_SECTION_START, MBAC_SECTION_END,
     PBAC_SECTION_START, PBAC_SECTION_END,
+    filter_real_stderr_lines,
     find_best_across_files, get_qm_corrections_files,
 )
 
@@ -339,8 +340,9 @@ def _get_energy_corrections(arkane_level_of_theory, bac_type: str | None) -> tup
                 'fi"',
             ]
             _, stderr = execute_command(command=commands, executable='/bin/bash')
-            if stderr:
-                logger.warning(f'get_qm_corrections.py stderr: {stderr}')
+            real_stderr = filter_real_stderr_lines(stderr) if stderr else []
+            if real_stderr:
+                logger.warning(f'get_qm_corrections.py stderr: {real_stderr}')
 
             result = read_yaml_file(tmp_out) or {}
             return result.get('aec'), result.get('bac')
@@ -412,8 +414,9 @@ def _compute_point_groups(species_dict: dict, project_directory: str) -> dict[st
             'fi"',
         ]
         _, stderr = execute_command(command=commands, executable='/bin/bash')
-        if stderr:
-            logger.warning(f'get_point_groups.py stderr: {stderr}')
+        real_stderr = filter_real_stderr_lines(stderr) if stderr else []
+        if real_stderr:
+            logger.warning(f'get_point_groups.py stderr: {real_stderr}')
 
         result = read_yaml_file(tmp_out) or {}
         return {str(k): (str(v) if v is not None else None) for k, v in result.items()}

arc/processor.py

@@ -10,6 +10,7 @@
 from arc.imports import settings
 from arc.level import Level
 from arc.job.local import execute_command
+from arc.statmech.arkane import filter_real_stderr_lines
 from arc.statmech.factory import statmech_factory
 
 
@@ -298,8 +299,9 @@ def compare_thermo(species_for_thermo_lib: list,
                 'fi"',
                 ]
     stdout, stderr = execute_command(command=commands, no_fail=True)
-    if len(stderr):
-        logger.error(f'Error while running RMG thermo script: {stderr}')
+    real_stderr = filter_real_stderr_lines(stderr) if stderr else []
+    if real_stderr:
+        logger.error(f'Error while running RMG thermo script: {real_stderr}')
     species_list = read_yaml_file(path=species_thermo_path)
     for original_spc, rmg_spc in zip(species_for_thermo_lib, species_list):
         h298, s298, comment = rmg_spc.get('h298', None), rmg_spc.get('s298', None), rmg_spc.get('comment', None)

arc/statmech/arkane.py

@@ -29,6 +29,60 @@
 RMG_ENV_NAME = settings.get('RMG_ENV_NAME', 'rmg_env')
 logger = get_logger()
 
+# Substrings that indicate a stderr line is harmless shell-init / library
+# noise rather than a real subprocess failure. Any line containing one of
+# these substrings is dropped by ``filter_real_stderr_lines``. Add new
+# patterns here when a benign emitter trips us up.
+_STDERR_NOISE_SUBSTRINGS = (
+    # Open Babel valence warnings around InChI generation.
+    "Open Babel Warning",
+    "Accepted unusual valence",
+    "==============================",
+    # JAX / TF startup chatter on some clusters.
+    "pjrt_executable.cc",
+    # Lmod (module system) "module load X" failures from the user's shell
+    # init when the named module was renamed/removed on the cluster. These
+    # don't break the spawned subprocess — Lmod just complains and moves on.
+    "Lmod has detected",
+    "module spider",
+    "module --ignore_cache",
+    "modulefiles written in TCL",
+    "#%Module",
+    "Please check the spelling or version number",
+    "It is also possible your cache file is out-of-date",
+    "Error while loading conda entry point",
+)
+
+
+def filter_real_stderr_lines(stderr):
+    """Drop harmless shell-init / library noise from a subprocess's stderr.
+
+    Accepts either a list of lines or a single multi-line string. Empty /
+    whitespace-only lines and bare quoted module names (e.g. ``"openmpi"``
+    on a Lmod-error line) are also dropped.
+
+    Returns a list of stripped lines that survived the filter — what's left
+    is genuine error output the caller should surface.
+    """
+    if isinstance(stderr, str):
+        lines = stderr.splitlines()
+    else:
+        lines = list(stderr)
+    real = []
+    for raw in lines:
+        line = raw.strip()
+        if not line:
+            continue
+        if any(s in line for s in _STDERR_NOISE_SUBSTRINGS):
+            continue
+        # Lmod prints the offending module name on its own line as a bare
+        # quoted token (e.g. `"openmpi"`). Drop those too.
+        if (line.startswith('"') and line.endswith('"') and len(line) <= 64
+                and ' ' not in line[1:-1]):
+            continue
+        real.append(line)
+    return real
+
 # Section boundary markers in the RMG quantum_corrections/data.py file.
 AEC_SECTION_START = "atom_energies = {"
 AEC_SECTION_END = "pbac = {"
@@ -505,8 +559,9 @@ def parse_arkane_thermo_output(self, statmech_dir: str) -> None:
                     'fi"',
                     ]
         stdout, stderr = execute_command(command=commands, executable='/bin/bash')
-        if len(stderr):
-            logger.error(f'Error while running Arkane thermo script:\n{stderr}')
+        real_stderr = filter_real_stderr_lines(stderr) if stderr else []
+        if real_stderr:
+            logger.error(f'Error while running Arkane thermo script:\n{real_stderr}')
         thermo_yaml_path = os.path.join(statmech_dir, 'thermo.yaml')
         if os.path.isfile(thermo_yaml_path):
             content = read_yaml_file(thermo_yaml_path) or {}
@@ -596,21 +651,7 @@ def run_arkane(statmech_dir: str) -> bool:
                                        no_fail=True,
                                        executable='/bin/bash')
     if std_err:
-        ignorable_phrases = [
-            "Open Babel Warning",
-            "Accepted unusual valence",
-            "==============================",
-            "pjrt_executable.cc",
-        ]
-
-        real_errors = []
-        for line in std_err:
-            line = line.strip()
-            if not line:
-                continue
-            if not any(phrase in line for phrase in ignorable_phrases):
-                real_errors.append(line)
-
+        real_errors = filter_real_stderr_lines(std_err)
         if real_errors:
             logger.info(f'Arkane run failed with errors:\n{std_err}')
             return False

arc/statmech/arkane_test.py

@@ -33,6 +33,7 @@
     _warn_no_match,
     check_arkane_aec,
     check_arkane_bacs,
+    filter_real_stderr_lines,
     get_arkane_model_chemistry,
 )
 from unittest.mock import patch
@@ -869,5 +870,108 @@ def test_dh_rxn_uses_kJmol_e_elect_for_composite_species(self):
         self.assertAlmostEqual(rxn.dh_rxn298, -1e5, places=6)
 
 
+class TestFilterRealStderrLines(unittest.TestCase):
+    """``filter_real_stderr_lines`` strips harmless shell-init / library noise
+    from a subprocess's stderr so ARC doesn't classify a successful Arkane run
+    as a failure."""
+
+    def test_open_babel_unusual_valence_warning_is_noise(self):
+        """Existing carve-out: Open Babel's InChI-code warnings are harmless."""
+        lines = [
+            "==============================",
+            "*** Open Babel Warning  in InChI code",
+            "  #1 :Accepted unusual valence(s): C(2)",
+            "==============================",
+        ]
+        self.assertEqual(filter_real_stderr_lines(lines), [])
+
+    def test_lmod_unknown_module_warning_is_noise(self):
+        """Regression: a stale ``module load openmpi`` in shell init prints a
+        Lmod block to stderr. ARC was treating it as a fatal Arkane failure
+        even though Arkane completed successfully."""
+        lines = [
+            'Lmod has detected the following error: The following module(s) are unknown:',
+            '"openmpi"',
+            '',
+            'Please check the spelling or version number. Also try "module spider ..."',
+            'It is also possible your cache file is out-of-date; it may help to try:',
+            '  $ module --ignore_cache load "openmpi"',
+            '',
+            'Also make sure that all modulefiles written in TCL start with the string',
+            '#%Module',
+        ]
+        self.assertEqual(filter_real_stderr_lines(lines), [])
+
+    def test_conda_libmamba_solver_load_failure_is_noise(self):
+        """A broken `_sqlite3` in the base conda (missing `sqlite3_deserialize`)
+        causes conda-libmamba-solver to fail loading and emit a stderr line on
+        every `conda run` invocation. The subprocess itself succeeds — conda
+        falls back to the classic solver — so this line is benign noise."""
+        lines = [
+            ('Error while loading conda entry point: conda-libmamba-solver '
+             '(/home/alon/miniconda3/lib/python3.11/lib-dynload/'
+             '_sqlite3.cpython-311-x86_64-linux-gnu.so: undefined symbol: '
+             'sqlite3_deserialize)'),
+        ] * 4
+        self.assertEqual(filter_real_stderr_lines(lines), [])
+
+    def test_conda_entry_point_noise_does_not_mask_real_error(self):
+        """Conda-entry-point noise is filtered, but a real traceback emitted by
+        the same script must still survive."""
+        lines = [
+            'Error while loading conda entry point: conda-libmamba-solver (...)',
+            'Traceback (most recent call last):',
+            'RuntimeError: thermo failed',
+        ]
+        result = filter_real_stderr_lines(lines)
+        self.assertEqual(
+            result,
+            ['Traceback (most recent call last):', 'RuntimeError: thermo failed'],
+        )
+
+    def test_real_error_is_preserved(self):
+        """A genuine traceback line passes through."""
+        lines = [
+            "==============================",
+            "*** Open Babel Warning",
+            "Traceback (most recent call last):",
+            '  File "arkane/main.py", line 123, in run',
+            "RuntimeError: Could not parse output",
+        ]
+        result = filter_real_stderr_lines(lines)
+        self.assertIn("Traceback (most recent call last):", result)
+        self.assertIn("RuntimeError: Could not parse output", result)
+        self.assertNotIn("==============================", result)
+
+    def test_mixed_lmod_and_real_error_keeps_only_real(self):
+        """When Lmod noise and a real error coexist, only the real error
+        survives. (The user's H/H2/OH composite SP failures should still
+        surface even if the shell init is noisy.)"""
+        lines = [
+            'Lmod has detected the following error: ...',
+            '"openmpi"',
+            '#%Module',
+            "AttributeError: 'NoneType' object has no attribute 'thermo'",
+        ]
+        result = filter_real_stderr_lines(lines)
+        self.assertEqual(
+            result,
+            ["AttributeError: 'NoneType' object has no attribute 'thermo'"],
+        )
+
+    def test_empty_and_whitespace_only_lines_dropped(self):
+        self.assertEqual(filter_real_stderr_lines(["", "   ", "\t"]), [])
+
+    def test_accepts_string_as_well_as_list(self):
+        """Some call sites pass a multiline string; the helper splits it."""
+        s = (
+            'Lmod has detected the following error: blah\n'
+            '"openmpi"\n'
+            '\n'
+            'Genuine error: foo\n'
+        )
+        self.assertEqual(filter_real_stderr_lines(s), ["Genuine error: foo"])
+
+
 if __name__ == '__main__':
     unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))