Route CLI status messages to stderr (#926)

Closes #858.

Dataset-setup notices, microsim "Running" / "completed" lines,
and "Results saved to ..." were all going to stdout, contaminating
piped CSV output in the default `policyengine-taxsim < input > output`
flow and in Daniel's `-o /dev/stdout` workflow. Send them to stderr
instead, matching the existing tqdm progress bars.

Stdout now contains only the CSV results; status output (including
tqdm) is exclusively on stderr. Users wanting full silence can pipe
stderr to /dev/null.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: PavelMakarchuk

changelog.d/cli-status-to-stderr.fixed.md

@@ -0,0 +1 @@
+Route CLI status messages (dataset setup, microsim progress, "Results saved") to stderr so they don't contaminate piped CSV output on stdout.

policyengine_taxsim/cli.py

@@ -197,13 +197,13 @@ def policyengine(input_file, output, logs, disable_salt, assume_w2_wages, sample
 
         # Generate YAML files if requested
         if logs:
-            click.echo("Generating PolicyEngine YAML test files...")
+            click.echo("Generating PolicyEngine YAML test files...", err=True)
             _generate_yaml_files(df_with_ids, results_df)
-            click.echo(f"Generated {len(df_with_ids)} YAML test files")
+            click.echo(f"Generated {len(df_with_ids)} YAML test files", err=True)
 
         # Save results to output file
         write_output(results_df, output)
-        click.echo(f"Results saved to {output}")
+        click.echo(f"Results saved to {output}", err=True)
 
     except Exception as e:
         click.echo(f"Error processing input: {str(e)}", err=True)

policyengine_taxsim/runners/base_runner.py

@@ -1,3 +1,5 @@
+import sys
+
 import pandas as pd
 from abc import ABC, abstractmethod
 from typing import Optional, Union
@@ -103,14 +105,14 @@ def save_results(
         """
         if results_df is None:
             if self.results is None:
-                print("Running calculations to generate results...")
+                print("Running calculations to generate results...", file=sys.stderr)
                 results_df = self.run()
             else:
                 results_df = self.results
 
         output_path = Path(output_path)
         write_output(results_df, output_path)
-        print(f"Results saved to: {output_path}")
+        print(f"Results saved to: {output_path}", file=sys.stderr)
 
     def get_record_count(self) -> int:
         """Get number of records in input data"""

policyengine_taxsim/runners/policyengine_runner.py

@@ -1,3 +1,5 @@
+import sys
+
 import pandas as pd
 import numpy as np
 import tempfile
@@ -733,7 +735,7 @@ def generate(self) -> None:
         self.input_df = self._ensure_required_columns(self.input_df)
 
         # Set defaults and convert TAXSIM32 format (vectorized)
-        print("Setting defaults for TAXSIM records...")
+        print("Setting defaults for TAXSIM records...", file=sys.stderr)
         self.input_df = self._apply_defaults_vectorized(self.input_df)
 
         # Extract years (assuming all records might have different years)
@@ -749,7 +751,7 @@ def generate(self) -> None:
         data = self._initialize_dataset_structure()
 
         # Process each year separately
-        print("Processing years for dataset generation...")
+        print("Processing years for dataset generation...", file=sys.stderr)
         for year in tqdm(unique_years, desc="Dataset generation by year"):
             year_mask = self.input_df["year"] == year
             year_data = self.input_df[year_mask].copy()
@@ -1005,7 +1007,8 @@ def run(self, show_progress: bool = True, on_progress=None) -> pd.DataFrame:
         """
         if show_progress:
             print(
-                f"Running PolicyEngine Microsimulation on {len(self.input_df)} records"
+                f"Running PolicyEngine Microsimulation on {len(self.input_df)} records",
+                file=sys.stderr,
             )
 
         # Ensure years are integers to handle decimal values like 2021.0
@@ -1043,7 +1046,7 @@ def run(self, show_progress: bool = True, on_progress=None) -> pd.DataFrame:
         results_df = pd.concat(frames, ignore_index=True)
 
         if show_progress:
-            print("PolicyEngine Microsimulation completed")
+            print("PolicyEngine Microsimulation completed", file=sys.stderr)
 
         return results_df