Command Output (#107)

* initial support for capturing output of commands
* adds wrapper integration tests, pass command through in capture_output for cmd
* updates env vars in api docs

Author: rsgalloway

docs/api.md

@@ -70,7 +70,9 @@ The following environment variables are used to help manage functionality:
 
 | Name | Description |
 |------|-------------|
-| DEFAULT_ENV_STACK | Name of the default environment stack (default) |
+| ALLOW_COMMANDS | Allow embedded commands |
+| COMMAND_TIMEOUT | Embedded command timeout in seconds |
+| DEFAULT_NAMESPACE | Name of the default environment stack (default) |
 | ENVPATH | Colon-separated paths to search for environment files |
 | IGNORE_MISSING | Ignore missing stack files when resolving environments |
 | INTERACTIVE | Force shells to run in interactive mode |

examples/default/test.env

@@ -0,0 +1,11 @@
+#!/usr/bin/env envstack
+include: [default]
+all: &all
+  PYVERSION: $(python -c "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')")
+  PYTHONPATH: ${DEPLOY_ROOT}/lib/python${PYVERSION}
+darwin:
+  <<: *all
+linux:
+  <<: *all
+windows:
+  <<: *all

lib/envstack/__init__.py

@@ -37,4 +37,4 @@
 __version__ = "0.9.6"
 
 from envstack.env import clear, init, revert, save  # noqa: F401
-from envstack.env import load_environ, resolve_environ  # noqa: F401
+from envstack.env import load_environ, resolve_environ  # noqa: F401

lib/envstack/config.py

@@ -57,12 +57,32 @@ def detect_shell():
             return "/usr/bin/bash"
 
 
+# debug mode
 DEBUG = os.getenv("DEBUG")
+
+# default namespace
 DEFAULT_NAMESPACE = os.getenv("DEFAULT_ENV_STACK", "default")
+
+# allow embedded commands
+ALLOW_COMMANDS = os.getenv("ALLOW_COMMANDS", "0") in ("1", "true", "True", "TRUE")
+
+# embedded command timeout in seconds
+try:
+    COMMAND_TIMEOUT = int(os.getenv("COMMAND_TIMEOUT", 5))
+except ValueError:
+    COMMAND_TIMEOUT = 5
+
+# default environment variables
 ENV = os.getenv("ENV", "prod")
 HOME = os.getenv("HOME")
-IGNORE_MISSING = bool(os.getenv("IGNORE_MISSING", 1))
+
+# Ignore missing stack files when resolving environments
+IGNORE_MISSING = os.getenv("IGNORE_MISSING", "1") in ("1", "true", "True", "TRUE")
+
+# logging level
 LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
+
+# platform and shell info
 ON_POSIX = "posix" in sys.builtin_module_names
 PLATFORM = platform.system().lower()
 PYTHON_VERSION = sys.version_info[0]

lib/envstack/util.py

@@ -61,6 +61,9 @@
     r"\$\{([a-zA-Z_][a-zA-Z0-9_]*)(?::([-=?])((?:\$\{[^}]+\}|[^}])*))?\}"
 )
 
+# regular expression pattern for command substitution
+cmdsub_pattern = re.compile(r"^\s*\$\((?P<cmd>.*)\)\s*$", re.DOTALL)
+
 
 def cache(func):
     """Function decorator to memoize return data."""
@@ -444,6 +447,9 @@ def substitute_variable(match):
                 )
 
             elif operator is None:
+                # check for command substitution
+                if config.ALLOW_COMMANDS and cmdsub_pattern.match(str(current)):
+                    return str(evaluate_command(current))
                 if is_literal(current):
                     return current  # file literal wins
                 if is_template(current):
@@ -493,7 +499,36 @@ def substitute_variable(match):
         else:
             result = expression
 
-    return sanitize_value(result)
+    resolved_value = sanitize_value(result)
+
+    # command substitution
+    if config.ALLOW_COMMANDS and isinstance(resolved_value, str):
+        resolved_value = evaluate_command(resolved_value)
+
+    return resolved_value
+
+
+def evaluate_command(command: str):
+    """
+    Evaluates command substitution in the given string.
+
+        PYVERSION: $(python --version)
+
+    :param command: The command string to evaluate.
+    :returns: The command output or original string if no command found.
+    """
+
+    from envstack.wrapper import capture_output
+
+    match = cmdsub_pattern.match(command)
+    if match:
+        exit_code, out, err = capture_output(match.group(1))
+        if exit_code == 0:
+            return out.strip()
+        else:
+            return err.strip() or null
+
+    return command
 
 
 def load_sys_path(

lib/envstack/wrapper.py

@@ -236,6 +236,68 @@ def get_subprocess_command(self, env):
         return list(self._subprocess_argv)
 
 
+def capture_output(
+    command: str,
+    namespace: str = config.DEFAULT_NAMESPACE,
+    timeout: int = config.COMMAND_TIMEOUT,
+):
+    """
+    Runs a command (string or argv) with the given stack namespace and captures stdout/stderr.
+
+    Returns: (returncode, stdout, stderr)
+    """
+    import errno
+
+    shellname = os.path.basename(config.SHELL).lower()
+    argv = list(command) if isinstance(command, (list, tuple)) else to_args(command)
+
+    # build env exactly like Wrapper.launch()
+    env = os.environ.copy()
+    env.update(resolve_environ(load_environ(namespace)))
+
+    # prefer argv execution where possible
+    if shellname in ["bash", "sh", "zsh"]:
+        needs_shell = any(re.search(r"\{(\w+)\}", a) for a in argv)
+        if needs_shell:
+            expr_argv = [re.sub(r"\{(\w+)\}", r"${\1}", a) for a in argv]
+            expr = shell_join(expr_argv)
+            cmd = [config.SHELL, "-c", expr]
+        else:
+            cmd = argv
+
+    # for cmd always use original command string
+    elif shellname in ["cmd"]:
+        cmd = command
+
+    else:
+        cmd = argv
+
+    try:
+        proc = subprocess.run(
+            cmd,
+            env=encode(env),
+            shell=False,
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return proc.returncode, proc.stdout, proc.stderr
+    except FileNotFoundError as e:
+        # no process ran; synthesize a bash-like error and code
+        # 127 is the conventional "command not found" code in shells
+        missing = e.filename or (
+            cmd[0] if isinstance(cmd, list) and cmd else str(command)
+        )
+        return 127, "", f"{missing}: command not found"
+    except OSError as e:
+        # Other OS-level execution errors (permission, exec format, etc.)
+        rc = 126 if getattr(e, "errno", None) in (errno.EACCES,) else 1
+        return rc, "", str(e)
+    except subprocess.TimeoutExpired as e:
+        return 124, "", f"Command timed out after {timeout} seconds"
+
+
 def run_command(command: str, namespace: str = config.DEFAULT_NAMESPACE):
     """
     Runs a given command with the given stack namespace.

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    integration: tests that run real subprocesses (slower / OS-dependent)

tests/fixtures/env/test.env

@@ -0,0 +1,11 @@
+#!/usr/bin/env envstack
+include: [default]
+all: &all
+  PYVERSION: $(python -c "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')")
+  PYTHONPATH: ${DEPLOY_ROOT}/lib/python${PYVERSION}
+darwin:
+  <<: *all
+linux:
+  <<: *all
+windows:
+  <<: *all

tests/test_cmds.py

@@ -99,7 +99,9 @@ def test_default(self):
 """
             % self.root
         )
-        output = subprocess.check_output(self.envstack_bin, shell=True, universal_newlines=True)
+        output = subprocess.check_output(
+            self.envstack_bin, shell=True, universal_newlines=True
+        )
         self.assertEqual(output, expected_output)
 
     def test_dev(self):
@@ -598,6 +600,21 @@ def test_project_echo_deploy_root_foobar(self):
         )
         self.assertEqual(output, expected_output)
 
+    def test_test_echo_pyversion(self):
+        """Tests the test stack with an echo command."""
+        command = "%s test -- echo {PYVERSION}" % self.envstack_bin
+        expected_output = f"{sys.version_info[0]}.{sys.version_info[1]}\n"
+        env = os.environ.copy()
+        env["ALLOW_COMMANDS"] = "1"
+        output = subprocess.check_output(
+            command,
+            start_new_session=True,
+            env=env,
+            shell=True,
+            universal_newlines=True,
+        )
+        self.assertEqual(output, expected_output)
+
 
 class TestSet(unittest.TestCase):
     """Tests various envstack set commands."""
@@ -791,15 +808,15 @@ def test_dev_deploy_root(self):
         )
         self.assertEqual(output, expected_output)
 
-    def test_test_deploy_root(self):
+    def test_project_deploy_root(self):
         command = "ENV=invalid %s project -- %s" % (self.envstack_bin, self.python_cmd)
         expected_output = f"{self.root}/project\n"
         output = subprocess.check_output(
             command, start_new_session=True, shell=True, universal_newlines=True
         )
         self.assertEqual(output, expected_output)
 
-    def test_foobar_deploy_root(self):
+    def test_project_foobar_deploy_root(self):
         command = "ENV=invalid %s project foobar -- %s" % (
             self.envstack_bin,
             self.python_cmd,

tests/test_env.py

@@ -805,7 +805,10 @@ def bake_encrypted_environ(self, stack_name):
             if key == "STACK":  # skip the stack name
                 continue
             encrypted_value = encrypted[key]
-            self.assertTrue(isinstance(encrypted_value, EncryptedNode), f"type is {type(encrypted_value)}")
+            self.assertTrue(
+                isinstance(encrypted_value, EncryptedNode),
+                f"type is {type(encrypted_value)}",
+            )
             self.assertEqual(encrypted_value.original_value, None)
             # self.assertNotEqual(encrypted_value.original_value, value)  # from_yaml only
             self.assertEqual(encrypted_value.value, value)
@@ -845,7 +848,9 @@ def resolve_encrypted_environ(self, stack_name):
             self.assertNotEqual(encrypted_value, value)
             self.assertNotEqual(encrypted_value, resolved_value)
             # self.assertEqual(resolved_value, encrypted_resolved_value)
-            if isinstance(resolved_value, str) and isinstance(encrypted_resolved_value, str):
+            if isinstance(resolved_value, str) and isinstance(
+                encrypted_resolved_value, str
+            ):
                 self.assertTrue(
                     resolved_value.startswith(encrypted_resolved_value),
                     f"{encrypted_resolved_value} not prefix of {resolved_value}",