Merge pull request #938 from Pipelex/release/v0.30.0

Release v0.30.0

Author: lchoquel

.badges/tests.json

@@ -1,7 +1,7 @@
 {
   "schemaVersion": 1,
   "label": "tests",
-  "message": "6095",
+  "message": "6111",
   "color": "blue",
   "cacheSeconds": 300
 }

.pipelex/pipelex.toml

@@ -161,7 +161,17 @@ default_directory_base_name = "pipeline"
 # Default logging level: "DEBUG", "INFO", "WARNING", "ERROR"
 default_log_level = "INFO"
 # Log output target: "stdout" or "stderr"
-console_log_target = "stdout"
+# console_log_target controls the RichHandler used by the logging system (log.debug/info/...).
+# Logs are diagnostics → stderr.
+console_log_target = "stderr"
+# console_print_target controls the Console returned by get_console(), used for banners,
+# deck notices, and the main `pipelex` CLI's data tables (`show backends`, `show models`,
+# `which`, `doctor`). Tables are data the user pipes to a file → stdout.
+# Note: the agent CLI (`pipelex-agent`) emits its actual JSON/markdown results via bare
+# builtin `print(...)` to sys.stdout, bypassing Rich entirely — so this knob does NOT
+# affect the agent CLI data channel. The agent CLI factory additionally forces both
+# targets to stderr internally to keep its stdout strictly reserved for the success
+# envelope.
 console_print_target = "stdout"
 
 [pipelex.log_config.package_log_levels]

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [v0.30.0] - 2026-05-25
+
+### Fixed
+
+- **`console_log_target` package default is now `stderr` (was `stdout`).** Logs now stay off the data channel by default, matching the intent of PR #452 ("default to stderr for outputs happening before initialization"). Downstream tooling that parses `pipelex` / `pipelex-agent` stdout as JSON (e.g. `mthds-js`'s `PipelexRunner`) is no longer at risk of stdout pollution from package-level logs — the bug was latent for stock installs because the agent-CLI JSON paths happen not to log at INFO+, but surfaced for anyone who raised `package_log_levels.pipelex` to DEBUG or added a setup-time log on the command path. The same flip is applied to the kit template (`pipelex/kit/configs/pipelex.toml`) that `pipelex init` copies to `~/.pipelex/`. **Note:** `console_print_target` is intentionally left at `stdout` — the main `pipelex` CLI emits human-facing tables (`show backends`, `show models`, `which`, `doctor`) via that channel, and downstream piping (`pipelex show backends > out.txt`) must keep working.
+
+- **`pipelex-agent` now pins both console targets to `stderr` regardless of user config.** `make_pipelex_for_agent_cli` injects `config_overrides` into `Pipelex.make()` that force `console_log_target = "stderr"` and `console_print_target = "stderr"` from the very first log/print fired during init, so a user override of either knob in `~/.pipelex/pipelex.toml` can no longer leak diagnostics onto the agent CLI's JSON data channel. Defense-in-depth post-init calls to `log.redirect_to_stderr()` and `get_pipelex_hub().set_console_print_target(STDERR)` remain. A new adversarial E2E test (`tests/e2e/agent_cli/test_stdout_is_clean_json.py::test_models_json_stdout_resists_user_targets_override_to_stdout`) pins the contract by overriding both targets to `stdout` and `package_log_levels.pipelex` to `DEBUG` and asserting `json.loads(stdout)` still parses.
+
 ## [v0.29.1] - 2026-05-21
 
 ### Fixed

pipelex/cli/agent_cli/commands/agent_cli_factory.py

@@ -1,13 +1,18 @@
 """Factory function for agent CLI commands -- JSON-only error output."""
 
 import warnings
+from collections.abc import Mapping
 from pathlib import Path
+from types import MappingProxyType
+from typing import Any
 
 import typer
 
 from pipelex.cli.agent_cli.commands.agent_output import agent_error, record_setup_warning
 from pipelex.cogt.exceptions import GatewayUnknownModelError, ModelDeckPresetValidatonError
+from pipelex.hub import PipelexHub
 from pipelex.pipelex import Pipelex
+from pipelex.system.console_target import ConsoleTarget
 from pipelex.system.pipelex_service.exceptions import (
     GatewayApiKeyMissingError,
     GatewayDoNotTrackConflictError,
@@ -23,6 +28,77 @@
 from pipelex.tools.log.log_levels import LogLevel
 from pipelex.tools.misc.pretty import PrettyPrinter, PrettyPrintMode
 
+# Canonical leaf dict for "for any agent-CLI invocation, both Rich-managed channels land
+# on stderr." Composed by two distinct call sites:
+#   - ``_AGENT_CLI_STDERR_CONSOLE_OVERRIDES`` below wraps it in the full-config-tree shape
+#     required by ``Pipelex.make(config_overrides=...)`` for the full-init path.
+#   - ``pipelex.cli.agent_cli.commands.doctor_cmd`` passes it flat to
+#     ``setup_doctor_runtime(log_config_overrides=...)`` for the doctor-only path that
+#     does not go through ``Pipelex.make``.
+#
+# These two knobs only control Rich-managed diagnostic channels — NOT the agent CLI's
+# data channel:
+#   - ``console_log_target``  -> the ``RichHandler`` for Python's logging system
+#                                (``log.debug/info/warning/error``).
+#   - ``console_print_target`` -> the ``Console`` returned by ``get_console()`` and used
+#                                for banners, deck notices, and the main ``pipelex`` CLI's
+#                                ``show backends`` / ``show models`` tables.
+#
+# The agent CLI's actual results (the JSON / markdown success envelope) are written by
+# ``agent_success`` / ``agent_success_formatted`` in ``agent_output.py`` via the bare
+# builtin ``print(...)`` straight to ``sys.stdout``. They do NOT go through Rich, so they
+# are unaffected by these overrides. Error envelopes go via ``print(..., file=sys.stderr)``
+# — also bypassing Rich.
+#
+# Pinning both targets to ``stderr`` therefore ensures that EVERYTHING ELSE Pipelex might
+# emit (setup-time ``log.debug`` from ``telemetry_factory.py``, a banner from
+# ``deck_notice.py``, any future ``get_console().print(...)`` on the agent CLI setup path)
+# lands on stderr regardless of what the user's ``~/.pipelex/pipelex.toml`` says — so the
+# JSON data channel on stdout stays clean for downstream consumers like ``mthds-js``'s
+# ``PipelexRunner`` that do ``JSON.parse(stdout)``.
+#
+# Wrapped in ``MappingProxyType`` so a stray ``AGENT_CLI_STDERR_LOG_FIELDS[...] = ...`` in
+# a future contributor's hot-fix raises immediately instead of silently mutating the
+# shared canonical instance (which both consumers below alias).
+AGENT_CLI_STDERR_LOG_FIELDS: Mapping[str, Any] = MappingProxyType(
+    {
+        "console_log_target": ConsoleTarget.STDERR,
+        "console_print_target": ConsoleTarget.STDERR,
+    }
+)
+
+# Direct alias: deep_update recurses into any ``Mapping`` and converts frozen leaves to
+# plain dicts at merge time, so referencing the canonical ``MappingProxyType`` here is
+# safe AND keeps mutation attempts on either reference loud (TypeError on the frozen
+# proxy) instead of silently diverging.
+_AGENT_CLI_STDERR_CONSOLE_OVERRIDES: dict[str, Any] = {
+    "pipelex": {"log_config": AGENT_CLI_STDERR_LOG_FIELDS},
+}
+
+
+def apply_agent_cli_output_discipline(log_level: LogLevel = LogLevel.WARNING) -> None:
+    """Pin pipelex log level, pretty-print silence, and hub console target to stderr.
+
+    Called from two paths:
+      - ``make_pipelex_for_agent_cli`` (full-init): defense-in-depth. ``Pipelex.__init__``
+        already pinned the hub console via ``set_console_print_target`` from the loaded
+        log_config (whose ``console_print_target`` was overridden to STDERR by
+        ``_AGENT_CLI_STDERR_CONSOLE_OVERRIDES``).
+      - ``agent_doctor_cmd`` (doctor-only): also defense-in-depth, since
+        ``setup_doctor_runtime`` now mirrors ``Pipelex.__init__`` and applies
+        ``set_console_print_target`` itself from the overridden log_config.
+
+    Safe to call from the broken-config doctor path where ``setup_doctor_runtime`` was
+    skipped: ``log.redirect_to_stderr`` no-ops when no rich_handler is registered, and
+    the hub print-target call is gated on a hub being installed.
+    """
+    log.set_level_for_package("pipelex", log_level)
+    log.redirect_to_stderr()
+    PrettyPrinter.mode = PrettyPrintMode.SILENT
+    hub = PipelexHub.get_optional_instance()
+    if hub is not None:
+        hub.set_console_print_target(target=ConsoleTarget.STDERR)
+
 
 def make_pipelex_for_agent_cli(
     library_dirs: list[str] | list[Path] | None = None,
@@ -41,6 +117,20 @@ def make_pipelex_for_agent_cli(
     human-readable markdown to stdout and exits 0, so the calling agent
     can display setup guidance directly.
 
+    Stdout contract: every ``pipelex-agent`` invocation reserves stdout exclusively
+    for the structured success envelope (JSON via ``--format json``, or markdown via
+    ``--format markdown``) emitted by ``agent_success`` / ``agent_success_formatted``.
+    All other output channels — logs, ``get_console().print(...)`` output, Rich pretty
+    prints, and ``agent_error`` envelopes — are pinned to stderr regardless of what the
+    user's ``pipelex.toml`` says. The factory enforces this via three layers:
+
+      1. ``config_overrides`` injected into ``Pipelex.make()`` pins both
+         ``console_log_target`` and ``console_print_target`` to ``stderr`` from the
+         very first log/print fired during init.
+      2. ``PrettyPrinter.mode = SILENT`` neutralizes ``pretty_print(...)`` entirely.
+      3. Post-init ``log.redirect_to_stderr()`` and
+         ``get_pipelex_hub().set_console_print_target(STDERR)`` defense-in-depth.
+
     Args:
         library_dirs: Optional library directories to use for the Pipelex instance.
         log_level: Log verbosity level (default WARNING for silent agent output).
@@ -58,7 +148,11 @@ def make_pipelex_for_agent_cli(
         with warnings.catch_warnings(record=True) as caught:
             warnings.simplefilter("always", RemoteConfigStaleWarning)
             pipelex_instance = Pipelex.make(
-                integration_mode=IntegrationMode.CLI, library_dirs=library_dirs, needs_inference=needs_inference, needs_model_specs=needs_model_specs
+                integration_mode=IntegrationMode.CLI,
+                library_dirs=library_dirs,
+                needs_inference=needs_inference,
+                needs_model_specs=needs_model_specs,
+                config_overrides=_AGENT_CLI_STDERR_CONSOLE_OVERRIDES,
             )
         # Surface a structured ``RemoteConfigStale`` entry so JSON consumers can react to
         # stale-cache operation without parsing stderr.
@@ -117,7 +211,5 @@ def make_pipelex_for_agent_cli(
 
     # Suppress Rich pretty-printing and INFO/DEV/DEBUG log noise so that agent
     # commands only emit structured JSON.  Warnings and errors still reach stderr.
-    PrettyPrinter.mode = PrettyPrintMode.SILENT
-    log.set_level_for_package("pipelex", log_level)
-    log.redirect_to_stderr()
+    apply_agent_cli_output_discipline(log_level=log_level)
     return pipelex_instance

pipelex/cli/agent_cli/commands/doctor_cmd.py

@@ -4,14 +4,18 @@
 
 import typer
 
+from pipelex.base_exceptions import PipelexConfigError
+from pipelex.cli.agent_cli.commands.agent_cli_factory import AGENT_CLI_STDERR_LOG_FIELDS, apply_agent_cli_output_discipline
 from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_error, agent_success, set_agent_cli_error_format
 from pipelex.cli.commands.doctor_cmd import (
+    BackendFileReport,
     ConfigLocationInfo,
     check_backend_credentials,
     check_config_files,
     check_models,
     check_telemetry_config,
     gather_config_location,
+    setup_doctor_runtime,
 )
 from pipelex.system.configuration.config_loader import config_manager
 
@@ -67,7 +71,11 @@ def _format_doctor_markdown(result: dict[str, Any]) -> str:
 
     # Models
     models_check = checks["models"]
-    lines.append(f"\n## Models \u2014 {_status_icon(models_check['healthy'])}\n")
+    models_skipped_flag = models_check.get("skipped", False)
+    # Skipped state renders with the warn icon so consumers don't confuse "deferred until
+    # config is fixed" with a genuine models failure.
+    models_icon = "\u26a0\ufe0f" if models_skipped_flag else _status_icon(models_check["healthy"])
+    lines.append(f"\n## Models \u2014 {models_icon}\n")
     lines.append(models_check["message"])
     for file_entry in models_check.get("backend_files", []):
         name = file_entry["backend_name"]
@@ -132,14 +140,59 @@ def agent_doctor_cmd(
         else:
             config_location = gather_config_location()
 
+        # Filesystem-only checks run BEFORE bootstrap: when no --global override is in
+        # play, setup_doctor_runtime's load_config materializes ~/.pipelex/ from kit
+        # templates as a side effect. Running it first would turn check_config_files into
+        # a silent installer on a fresh machine. (The --global path skips materialization
+        # — see load_config.)
         config_healthy, config_missing_count, config_message = check_config_files(config_dir=config_dir)
         telemetry_healthy, telemetry_message = check_telemetry_config(config_dir=config_dir)
         backends_healthy, backend_credential_reports, backends_message = check_backend_credentials(config_dir=config_dir)
-        models_healthy, models_message, backend_file_reports = check_models(config_dir=config_dir)
+
+        # check_models requires the hub + log.configure produced by setup_doctor_runtime.
+        # When the config is broken (either shape-check fails up front or full validation
+        # raises PipelexConfigError inside the bootstrap), we skip running check_models and
+        # mark models as skipped — keeping the partial check tuples gathered above so the
+        # JSON envelope still reports telemetry/backends instead of degrading to a single
+        # error payload.
+        models_skipped: bool = False
+        models_healthy: bool
+        models_message: str
+        backend_file_reports: dict[str, BackendFileReport]
+        if config_healthy:
+            try:
+                setup_doctor_runtime(log_config_overrides=AGENT_CLI_STDERR_LOG_FIELDS, config_dir=config_dir)
+                # Pin discipline BEFORE check_models. setup_doctor_runtime uses
+                # log.configure_if_unset(), which no-ops when a prior process already configured
+                # logging (embedded reuse, interleaved tests) — in that case AGENT_CLI_STDERR_LOG_FIELDS
+                # never reaches the handler, and any log line check_models emits could land on stdout.
+                # apply_agent_cli_output_discipline mutates the existing handler unconditionally via
+                # log.redirect_to_stderr, closing that window before any check fires.
+                apply_agent_cli_output_discipline()
+                models_healthy, models_message, backend_file_reports = check_models(config_dir=config_dir)
+            except PipelexConfigError as exc:
+                # A config can pass check_config_files's shape check and still fail full validation
+                # inside setup_doctor_runtime (e.g. a layered override file). Surface the translated
+                # message via the same partial-report shape as the broken-config branch.
+                models_skipped = True
+                models_healthy = False
+                models_message = f"skipped — {exc.message}"
+                backend_file_reports = {}
+        else:
+            models_skipped = True
+            models_healthy = False
+            models_message = "skipped — fix configuration errors first"
+            backend_file_reports = {}
     except Exception as exc:  # noqa: BLE001
-        # Agent CLI command boundary: agent_error() (NoReturn) converts any unexpected failure into the structured error payload.
+        # Agent CLI command boundary: agent_error() (NoReturn) converts any genuinely
+        # unexpected failure into the structured error payload. PipelexConfigError is
+        # handled by the inner arm above and never reaches here.
         agent_error(f"Health check failed unexpectedly: {exc}", type(exc).__name__, cause=exc)
 
+    # Pin stdout discipline regardless of bootstrap path (broken-config branch never
+    # installed a hub or configured log — the helper guards both internally).
+    apply_agent_cli_output_discipline()
+
     all_healthy = config_healthy and telemetry_healthy and backends_healthy and models_healthy
 
     # Build backend credential details
@@ -215,6 +268,7 @@ def agent_doctor_cmd(
             },
             "models": {
                 "healthy": models_healthy,
+                "skipped": models_skipped,
                 "message": models_message,
                 "backend_files": backend_files_list,
             },