Clearer error when a job hits a full disk (ENOSPC) (#1899)

cwltool/job.py

@@ -1,4 +1,5 @@
 import datetime
+import errno
 import functools
 import itertools
 import logging
@@ -376,7 +377,7 @@ def stderr_stdout_log_path(
             outputs = self.collect_outputs(self.outdir, rcode)
             outputs = bytes2str_in_dicts(outputs)  # type: ignore
         except OSError as e:
-            if e.errno == 2:
+            if e.errno == errno.ENOENT:
                 if runtime:
                     _logger.error(
                         "'%s' not found: %s", runtime[0], str(e), exc_info=runtimeContext.debug
@@ -388,6 +389,16 @@ def stderr_stdout_log_path(
                         str(e),
                         exc_info=runtimeContext.debug,
                     )
+            elif e.errno == errno.ENOSPC:
+                _logger.error(
+                    "[job %s] No space left on device. The temporary directory (%s) "
+                    "and/or output directory (%s) may be full; free up space, or point "
+                    "--tmpdir-prefix and --outdir at a location with more capacity.",
+                    self.name,
+                    self.tmpdir,
+                    self.outdir,
+                    exc_info=runtimeContext.debug,
+                )
             else:
                 _logger.exception(
                     "Exception while running job: %s", str(e), exc_info=runtimeContext.debug

tests/test_examples.py

@@ -1,3 +1,4 @@
+import errno
 import json
 import logging
 import os
@@ -1593,6 +1594,22 @@ def test_bad_basecommand(factor: str) -> None:
     assert error_code == 1
 
 
+def test_disk_full_error_message(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A full disk (ENOSPC) while running a job yields a clear message, not a traceback."""
+
+    def _raise_enospc(obj: Any) -> Any:
+        raise OSError(errno.ENOSPC, "No space left on device")
+
+    # Inject ENOSPC at the output-handling stage of a normal job run.
+    monkeypatch.setattr("cwltool.job.bytes2str_in_dicts", _raise_enospc)
+    error_code, stdout, stderr = get_main_output([get_data("tests/echo.cwl"), "--inp", "hello"])
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "No space left on device" in stderr, stderr
+    assert "--tmpdir-prefix" in stderr, stderr
+    assert "Traceback (most recent call last)" not in stderr, stderr
+    assert error_code == 1
+
+
 @needs_docker
 @pytest.mark.parametrize("factor", test_factors)
 def test_bad_basecommand_docker(factor: str) -> None: