From 5a0de46acb14c1c3f6ca296122d24013ead690d2 Mon Sep 17 00:00:00 2001 From: marcellodebernardi Date: Fri, 27 Feb 2026 14:58:27 +0000 Subject: [PATCH 1/3] feat: add model card packaging --- plexe/CODE_INDEX.md | 9 +- .../packaging/model_card_template.py | 575 ++++++++++++++++++ plexe/workflow.py | 13 + tests/CODE_INDEX.md | 10 +- tests/unit/workflow/__init__.py | 1 + tests/unit/workflow/test_model_card.py | 225 +++++++ 6 files changed, 831 insertions(+), 2 deletions(-) create mode 100644 plexe/templates/packaging/model_card_template.py create mode 100644 tests/unit/workflow/__init__.py create mode 100644 tests/unit/workflow/test_model_card.py diff --git a/plexe/CODE_INDEX.md b/plexe/CODE_INDEX.md index 3315ae54..004a981e 100644 --- a/plexe/CODE_INDEX.md +++ b/plexe/CODE_INDEX.md @@ -477,6 +477,13 @@ Standard XGBoost predictor - NO Plexe dependencies. - `__init__(self, model_dir: str)` - `predict(self, x: pd.DataFrame) -> pd.DataFrame` - Make predictions on input DataFrame. +--- +## `templates/packaging/model_card_template.py` +Model card template generator. + +**Functions:** +- `generate_model_card(context, final_metrics: dict, evaluation_report: Any | None) -> str` - Generate a Markdown model card for the final package. + --- ## `templates/training/train_catboost.py` Hardcoded robust CatBoost training loop. @@ -700,4 +707,4 @@ Main workflow orchestrator. - `evaluate_final(spark: SparkSession, context: BuildContext, solution: Solution, config: Config, on_checkpoint_saved: Callable[[str, Path, Path], None] | None) -> dict` - Phase 5: Final evaluation on test set sample. - `package_final_model(spark: SparkSession, context: BuildContext, solution: Solution, final_metrics: dict, on_checkpoint_saved: Callable[[str, Path, Path], None] | None) -> Path` - Package all final deliverables into a unified directory. ---- \ No newline at end of file +--- diff --git a/plexe/templates/packaging/model_card_template.py b/plexe/templates/packaging/model_card_template.py new file mode 100644 index 00000000..3486dbed --- /dev/null +++ b/plexe/templates/packaging/model_card_template.py @@ -0,0 +1,575 @@ +""" +Model card template generator. + +Builds a comprehensive MODEL_CARD.md from packaged artifacts and evaluation outputs. +""" + +from __future__ import annotations + +import json +import numbers +import re +from pathlib import Path +from typing import Any + +import yaml + + +MODEL_CARD_TITLE = "Model Card" +DEFAULT_NOT_AVAILABLE = "Not available" + + +def generate_model_card(context, final_metrics: dict, evaluation_report: Any | None) -> str: + """ + Generate a Markdown model card for the final package. + + Args: + context: BuildContext + final_metrics: Final metrics dict from evaluation + evaluation_report: EvaluationReport or dict (optional) + + Returns: + Markdown string + """ + package_dir = Path(context.work_dir) / "model" + + model_metadata = _safe_load_yaml(package_dir / "model.yaml") or {} + input_schema = _safe_load_json(package_dir / "schemas" / "input.json") or {} + hyperparameters = _safe_load_json(package_dir / "config" / "hyperparameters.json") or {} + evaluation_data = _normalize_evaluation_report(evaluation_report, package_dir) or {} + + task_analysis = context.task_analysis or {} + stats = context.stats or {} + + model_type = model_metadata.get("model_type") or DEFAULT_NOT_AVAILABLE + task_type = model_metadata.get("task_type") or task_analysis.get("task_type") or DEFAULT_NOT_AVAILABLE + + metric_name = _resolve_primary_metric_name(context, final_metrics, evaluation_data) + metric_value = _resolve_primary_metric_value(final_metrics, evaluation_data) + + lines: list[str] = [] + lines.append(f"# {MODEL_CARD_TITLE}") + lines.append("") + + # Summary + lines.append("## Summary") + lines.append(f"- Intent: {context.intent or DEFAULT_NOT_AVAILABLE}") + lines.append(f"- Task type: {task_type}") + lines.append(f"- Model type: {model_type}") + lines.append(f"- Primary metric ({metric_name}): {_format_metric(metric_value)}") + lines.append("") + + # Dataset + lines.append("## Dataset") + train_samples = _get_nested(model_metadata, ["training", "train_samples"]) + val_samples = _get_nested(model_metadata, ["training", "val_samples"]) + features_count = _resolve_feature_count(input_schema, model_metadata, task_analysis) + test_samples = final_metrics.get("test_samples") + + lines.append(f"- Training samples: {_format_count(train_samples)}") + lines.append(f"- Validation samples: {_format_count(val_samples)}") + lines.append(f"- Test samples: {_format_count(test_samples)}") + lines.append(f"- Features: {_format_count(features_count)}") + + notable_characteristics = _collect_notable_characteristics(task_analysis, stats) + if notable_characteristics: + lines.append("Notable data characteristics:") + for item in notable_characteristics: + lines.append(f"- {item}") + lines.append("") + + # Features Used + lines.append("## Features Used") + features = _resolve_features(input_schema, task_analysis) + if features: + lines.append(f"Input features ({len(features)}):") + for feature in features: + lines.append(f"- `{feature}`") + else: + lines.append(DEFAULT_NOT_AVAILABLE) + + explainability = _get_explainability_report(evaluation_data) + feature_importance = _get_feature_importance(explainability) + if feature_importance: + method_used = _get_value(explainability, "method_used") + if method_used: + lines.append("") + lines.append(f"Feature importance ({method_used}):") + else: + lines.append("") + lines.append("Feature importance:") + lines.extend(_format_feature_importance_table(feature_importance, explainability)) + + lines.append("") + + # Excluded Columns + excluded_columns = getattr(context, "excluded_columns", []) + excluded_lines = _format_excluded_columns(excluded_columns) + if excluded_lines: + lines.append("## Excluded Columns") + lines.extend(excluded_lines) + lines.append("") + + # Performance + lines.append("## Performance") + lines.append(f"- Primary metric ({metric_name}): {_format_metric(metric_value)}") + baseline_lines = _format_baseline_comparison( + metric_name=metric_name, + metric_value=metric_value, + evaluation_data=evaluation_data, + context=context, + model_metadata=model_metadata, + ) + if baseline_lines: + lines.extend(baseline_lines) + + additional_metrics = _collect_additional_metrics(metric_name, final_metrics, evaluation_data) + if additional_metrics: + lines.append("Additional metrics:") + lines.extend(_format_metrics_table(additional_metrics)) + else: + lines.append("Additional metrics: Not available") + lines.append("") + + # Evaluation Verdict + lines.append("## Evaluation Verdict") + if evaluation_data: + verdict = evaluation_data.get("verdict") or DEFAULT_NOT_AVAILABLE + deployment_ready = evaluation_data.get("deployment_ready") + summary = evaluation_data.get("summary") or DEFAULT_NOT_AVAILABLE + key_concerns = evaluation_data.get("key_concerns") or [] + + lines.append(f"- Verdict: {verdict}") + lines.append( + f"- Deployment ready: {deployment_ready if deployment_ready is not None else DEFAULT_NOT_AVAILABLE}" + ) + lines.append(f"- Summary: {summary}") + if key_concerns: + lines.append("Key concerns:") + for concern in key_concerns: + lines.append(f"- {concern}") + + recommendations = evaluation_data.get("recommendations") or [] + if recommendations: + lines.append("Recommendations:") + for rec in recommendations: + lines.append(f"- {_format_recommendation(rec)}") + else: + lines.append(DEFAULT_NOT_AVAILABLE) + lines.append("") + + # Hyperparameters + lines.append("## Hyperparameters") + if hyperparameters: + lines.extend(_format_hyperparameters_table(hyperparameters)) + else: + lines.append(DEFAULT_NOT_AVAILABLE) + lines.append("") + + # Known Limitations + lines.append("## Known Limitations") + data_challenges = task_analysis.get("data_challenges") or [] + recommendations = evaluation_data.get("recommendations") if evaluation_data else [] + + if data_challenges: + lines.append("Data challenges:") + for challenge in data_challenges: + lines.append(f"- {challenge}") + if recommendations: + lines.append("Evaluation recommendations:") + for rec in recommendations: + lines.append(f"- {_format_recommendation(rec)}") + + if not data_challenges and not recommendations: + lines.append(DEFAULT_NOT_AVAILABLE) + lines.append("") + + # Reproducibility + lines.append("## Reproducibility") + metadata = model_metadata.get("metadata", {}) + training_timestamp = metadata.get("created_at") or DEFAULT_NOT_AVAILABLE + experiment_id = metadata.get("experiment_id") or context.experiment_id or DEFAULT_NOT_AVAILABLE + builder_version = _resolve_model_builder_version() or metadata.get("version") or DEFAULT_NOT_AVAILABLE + + lines.append(f"- Experiment ID: {experiment_id}") + lines.append(f"- Training timestamp: {training_timestamp}") + lines.append(f"- Model-builder-v2 version: {builder_version}") + + return "\n".join(lines).strip() + "\n" + + +# ============================================ +# Helpers +# ============================================ + + +def _safe_load_json(path: Path) -> dict | None: + if not path.exists(): + return None + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError, UnicodeDecodeError): + return None + + +def _safe_load_yaml(path: Path) -> dict | None: + if not path.exists(): + return None + try: + with open(path, encoding="utf-8") as f: + return yaml.safe_load(f) + except (yaml.YAMLError, OSError, UnicodeDecodeError): + return None + + +def _normalize_evaluation_report(evaluation_report: Any | None, package_dir: Path) -> dict | None: + if evaluation_report is None: + fallback = _safe_load_json(package_dir / "evaluation" / "reports" / "evaluation.json") + return fallback + + if isinstance(evaluation_report, dict): + return evaluation_report + + if hasattr(evaluation_report, "to_dict"): + try: + return evaluation_report.to_dict() + except Exception: + pass + + if hasattr(evaluation_report, "__dict__"): + try: + return dict(evaluation_report.__dict__) + except Exception: + return None + + return None + + +def _get_nested(data: dict, keys: list[str]) -> Any: + current: Any = data + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + if current is None: + return None + return current + + +def _get_value(data: Any, key: str) -> Any: + if data is None: + return None + if isinstance(data, dict): + return data.get(key) + return getattr(data, key, None) + + +def _resolve_primary_metric_name(context, final_metrics: dict, evaluation_data: dict) -> str: + core_metrics = evaluation_data.get("core_metrics") if evaluation_data else None + metric_name = None + if isinstance(core_metrics, dict): + metric_name = core_metrics.get("primary_metric_name") + if not metric_name and context.metric: + metric_name = context.metric.name + if not metric_name: + metric_name = final_metrics.get("metric") + return metric_name or DEFAULT_NOT_AVAILABLE + + +def _resolve_primary_metric_value(final_metrics: dict, evaluation_data: dict) -> Any: + core_metrics = evaluation_data.get("core_metrics") if evaluation_data else None + if isinstance(core_metrics, dict) and core_metrics.get("primary_metric_value") is not None: + return core_metrics.get("primary_metric_value") + return final_metrics.get("performance") + + +def _resolve_feature_count(input_schema: dict, model_metadata: dict, task_analysis: dict) -> int | None: + if input_schema.get("properties"): + return len(input_schema.get("properties", {})) + count = _get_nested(model_metadata, ["training", "features_count"]) + if count is not None: + return count + input_description = task_analysis.get("input_description") or {} + if isinstance(input_description, dict): + return input_description.get("num_features") + return None + + +def _resolve_features(input_schema: dict, task_analysis: dict) -> list[str]: + if input_schema.get("properties"): + return sorted(input_schema.get("properties", {}).keys()) + input_description = task_analysis.get("input_description") or {} + if isinstance(input_description, dict): + columns = input_description.get("feature_columns") or input_description.get("columns") + if isinstance(columns, list): + return columns + return [] + + +def _collect_notable_characteristics(task_analysis: dict, stats: dict) -> list[str]: + notable = [] + key_insights = task_analysis.get("key_insights") or [] + if isinstance(key_insights, list): + notable.extend(str(item) for item in key_insights if item) + + quality_issues = stats.get("quality_issues") or [] + if isinstance(quality_issues, list): + notable.extend(str(item) for item in quality_issues if item) + + input_description = task_analysis.get("input_description") + if not notable and isinstance(input_description, dict): + summary = json.dumps(input_description, ensure_ascii=True) + notable.append(f"Input description: {summary}") + + return notable + + +def _get_explainability_report(evaluation_data: dict) -> dict | None: + if not evaluation_data: + return None + return evaluation_data.get("explainability") or evaluation_data.get("explainability_report") + + +def _get_feature_importance(explainability: dict | None) -> dict | None: + if not explainability: + return None + feature_importance = explainability.get("feature_importance") + if isinstance(feature_importance, dict) and feature_importance: + return feature_importance + return None + + +def _format_feature_importance_table(feature_importance: dict, explainability: dict | None) -> list[str]: + lines = ["| Feature | Importance |", "| --- | --- |"] + top_features = [] + if explainability: + top_features = explainability.get("top_features") or [] + + if top_features: + for feature in top_features: + value = feature_importance.get(feature) + lines.append(f"| `{feature}` | {_format_metric(value)} |") + return lines + + for feature, value in sorted( + feature_importance.items(), + key=lambda item: _feature_importance_sort_key(item[1]), + reverse=True, + ): + lines.append(f"| `{feature}` | {_format_metric(value)} |") + + return lines + + +def _feature_importance_sort_key(value: Any) -> float: + if _is_number(value): + return float(value) + return float("-inf") + + +def _format_excluded_columns(excluded_columns: Any) -> list[str]: + if not excluded_columns: + return [] + + lines: list[str] = [] + if isinstance(excluded_columns, dict): + for col, reason in excluded_columns.items(): + reason_text = _format_reason(reason) + lines.append(_format_excluded_line(col, reason_text)) + elif isinstance(excluded_columns, list): + for item in excluded_columns: + if isinstance(item, dict): + col = item.get("column") or item.get("name") or item.get("feature") or item.get("col") + reason = item.get("reason") or item.get("issue") or item.get("notes") or item.get("why") + if col: + reason_text = _format_reason(reason) + lines.append(_format_excluded_line(col, reason_text)) + else: + lines.append(f"- {json.dumps(item, ensure_ascii=True)}") + else: + lines.append(f"- `{item}`") + else: + lines.append(f"- `{excluded_columns}`") + + return lines + + +def _format_excluded_line(column: str, reason: str | None) -> str: + if reason: + return f"- `{column}` - {reason}" + return f"- `{column}`" + + +def _format_reason(reason: Any) -> str | None: + if reason is None: + return None + if isinstance(reason, list): + return "; ".join(str(item) for item in reason if item) + return str(reason) + + +def _format_baseline_comparison( + metric_name: str, + metric_value: Any, + evaluation_data: dict, + context, + model_metadata: dict, +) -> list[str]: + baseline_value = None + baseline_name = None + + baseline_comparison = evaluation_data.get("baseline_comparison") if evaluation_data else None + if isinstance(baseline_comparison, dict): + baseline_name = baseline_comparison.get("baseline_name") + baseline_performance = baseline_comparison.get("baseline_performance") or {} + if isinstance(baseline_performance, dict): + baseline_value = baseline_performance.get(metric_name) + + if baseline_value is None: + baseline_value = _get_nested(model_metadata, ["metric", "baseline"]) + + if baseline_value is None and getattr(context, "heuristic_baseline", None): + baseline_value = context.heuristic_baseline.performance + baseline_name = context.heuristic_baseline.name + + if baseline_value is None or metric_value is None: + return [] + + metric_direction = _get_nested(model_metadata, ["metric", "optimization_direction"]) + if not metric_direction and getattr(context, "metric", None): + metric_direction = context.metric.optimization_direction + + improvement = _calculate_improvement(metric_value, baseline_value, metric_direction) + pct_improvement = _calculate_percent_improvement(improvement, baseline_value) + + baseline_label = f" ({baseline_name})" if baseline_name else "" + + lines = [ + f"- Baseline{baseline_label}: {_format_metric(baseline_value)}", + f"- Improvement over baseline: {_format_metric(improvement)}" + + (f" ({_format_percent(pct_improvement)})" if pct_improvement is not None else ""), + ] + + return lines + + +def _calculate_improvement(metric_value: Any, baseline_value: Any, direction: str | None) -> float | None: + if not _is_number(metric_value) or not _is_number(baseline_value): + return None + + if direction == "lower": + return float(baseline_value) - float(metric_value) + return float(metric_value) - float(baseline_value) + + +def _calculate_percent_improvement(improvement: float | None, baseline_value: Any) -> float | None: + if improvement is None or not _is_number(baseline_value): + return None + if float(baseline_value) == 0: + return None + return (improvement / abs(float(baseline_value))) * 100 + + +def _collect_additional_metrics(metric_name: str, final_metrics: dict, evaluation_data: dict) -> dict: + core_metrics = evaluation_data.get("core_metrics") if evaluation_data else None + all_metrics = {} + + if isinstance(core_metrics, dict): + all_metrics.update(core_metrics.get("all_metrics") or {}) + if not all_metrics: + all_metrics.update(final_metrics.get("all_metrics") or {}) + + if metric_name in all_metrics: + all_metrics = {k: v for k, v in all_metrics.items() if k != metric_name} + + return all_metrics + + +def _format_metrics_table(metrics: dict) -> list[str]: + lines = ["| Metric | Value |", "| --- | --- |"] + for name, value in sorted(metrics.items()): + lines.append(f"| `{name}` | {_format_metric(value)} |") + return lines + + +def _format_hyperparameters_table(hyperparameters: dict) -> list[str]: + lines = ["| Hyperparameter | Value |", "| --- | --- |"] + for key in sorted(hyperparameters.keys()): + value = hyperparameters[key] + lines.append(f"| `{key}` | `{_stringify(value)}` |") + return lines + + +def _format_recommendation(recommendation: Any) -> str: + if isinstance(recommendation, dict): + priority = recommendation.get("priority") + action = recommendation.get("action") + rationale = recommendation.get("rationale") + if priority and action and rationale: + return f"{priority}: {action} - {rationale}" + if action and rationale: + return f"{action} - {rationale}" + if action: + return str(action) + return str(recommendation) + + +def _stringify(value: Any) -> str: + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=True) + return str(value) + + +def _format_metric(value: Any) -> str: + if value is None: + return DEFAULT_NOT_AVAILABLE + if _is_number(value): + return f"{float(value):.4f}" + return str(value) + + +def _format_count(value: Any) -> str: + if value is None: + return DEFAULT_NOT_AVAILABLE + if _is_number(value): + if float(value).is_integer(): + return str(int(value)) + return str(value) + return str(value) + + +def _format_percent(value: float | None) -> str: + if value is None: + return DEFAULT_NOT_AVAILABLE + return f"{value:.2f}%" + + +def _is_number(value: Any) -> bool: + return isinstance(value, numbers.Real) + + +def _resolve_model_builder_version() -> str | None: + pyproject_path = _find_pyproject() + if not pyproject_path: + return None + + try: + content = pyproject_path.read_text() + except Exception: + return None + + match = re.search(r"^version\s*=\s*\"([^\"]+)\"", content, re.MULTILINE) + if match: + return match.group(1) + return None + + +def _find_pyproject() -> Path | None: + current = Path(__file__).resolve() + for _ in range(6): + candidate = current.parent / "pyproject.toml" + if candidate.exists(): + return candidate + current = current.parent + return None diff --git a/plexe/workflow.py b/plexe/workflow.py index 3f122741..efac9400 100644 --- a/plexe/workflow.py +++ b/plexe/workflow.py @@ -61,6 +61,7 @@ from plexe.utils.reporting import save_report from plexe.templates.features.pipeline_fitter import fit_pipeline from plexe.templates.features.pipeline_runner import transform_dataset_via_spark +from plexe.templates.packaging.model_card_template import generate_model_card from plexe.helpers import evaluate_on_sample, select_viable_model_types logger = logging.getLogger(__name__) @@ -2016,6 +2017,18 @@ def package_final_model( # Note: Keep artifacts/metadata.json for debugging (contains full training metadata) # Hyperparameters are duplicated in config/ for convenience + # ============================================ + # Step 4b: Create MODEL_CARD.md + # ============================================ + logger.info("Creating MODEL_CARD.md...") + + evaluation_report = context.scratch.get("_evaluation_report") + model_card_content = generate_model_card(context, final_metrics, evaluation_report) + model_card_path = package_dir / "MODEL_CARD.md" + with open(model_card_path, "w", encoding="utf-8") as f: + f.write(model_card_content) + logger.info(" Created MODEL_CARD.md") + # ============================================ # Step 5: Create Tarball Archive # ============================================ diff --git a/tests/CODE_INDEX.md b/tests/CODE_INDEX.md index 547e0395..f05f2080 100644 --- a/tests/CODE_INDEX.md +++ b/tests/CODE_INDEX.md @@ -169,4 +169,12 @@ Unit tests for validation functions. - `test_validate_metric_function_object_success()` - Callable with correct signature should pass. - `test_validate_metric_function_object_bad_signature()` - Callable with wrong arg names should fail. ---- \ No newline at end of file +--- +## `unit/workflow/test_model_card.py` +Unit tests for model card generation. + +**Functions:** +- `test_generate_model_card_full_context(tmp_path: Path) -> None` - No description +- `test_generate_model_card_minimal_context(tmp_path: Path) -> None` - No description + +--- diff --git a/tests/unit/workflow/__init__.py b/tests/unit/workflow/__init__.py new file mode 100644 index 00000000..94998dc7 --- /dev/null +++ b/tests/unit/workflow/__init__.py @@ -0,0 +1 @@ +"""Workflow unit tests.""" diff --git a/tests/unit/workflow/test_model_card.py b/tests/unit/workflow/test_model_card.py new file mode 100644 index 00000000..0f4b4938 --- /dev/null +++ b/tests/unit/workflow/test_model_card.py @@ -0,0 +1,225 @@ +"""Unit tests for model card generation.""" + +import json +from pathlib import Path + +import yaml + +from plexe.models import ( + Baseline, + BaselineComparisonReport, + BuildContext, + CoreMetricsReport, + DiagnosticReport, + EvaluationReport, + ExplainabilityReport, + Metric, + RobustnessReport, +) +from plexe.templates.packaging.model_card_template import generate_model_card + + +def _write_package_files(work_dir: Path) -> None: + package_dir = work_dir / "model" + (package_dir / "schemas").mkdir(parents=True, exist_ok=True) + (package_dir / "config").mkdir(parents=True, exist_ok=True) + + input_schema = { + "type": "object", + "properties": { + "age": {"type": "number"}, + "tenure": {"type": "number"}, + "balance": {"type": "number"}, + }, + "required": ["age", "tenure", "balance"], + } + (package_dir / "schemas" / "input.json").write_text( + json.dumps(input_schema), + encoding="utf-8", + ) + + hyperparameters = {"max_depth": 6, "learning_rate": 0.1, "subsample": 0.8} + (package_dir / "config" / "hyperparameters.json").write_text( + json.dumps(hyperparameters), + encoding="utf-8", + ) + + model_metadata = { + "model_format": "plexe_v1", + "intent": "Predict churn", + "model_type": "xgboost", + "task_type": "binary_classification", + "target_column": "churn", + "output_targets": ["churn"], + "metric": { + "name": "roc_auc", + "value": 0.85, + "optimization_direction": "higher", + "baseline": 0.6, + }, + "training": {"features_count": 3, "train_samples": 800, "val_samples": 100}, + "metadata": { + "created_at": "2025-01-01T00:00:00Z", + "experiment_id": "exp_001", + "user_id": "user_123", + "trained_by": "plexe", + }, + } + (package_dir / "model.yaml").write_text( + yaml.safe_dump(model_metadata, sort_keys=False), + encoding="utf-8", + ) + + +def _make_full_context(work_dir: Path) -> BuildContext: + context = BuildContext( + user_id="user_123", + experiment_id="exp_001", + dataset_uri="s3://bucket/train.parquet", + work_dir=work_dir, + intent="Predict churn", + ) + context.metric = Metric(name="roc_auc", optimization_direction="higher") + context.task_analysis = { + "task_type": "binary_classification", + "data_challenges": ["class imbalance", "missing values"], + "key_insights": ["Age correlates with churn", "Balance is highly predictive"], + "input_description": { + "type": "tabular", + "num_features": 3, + "feature_columns": ["age", "tenure", "balance"], + }, + } + context.stats = { + "total_rows": 1000, + "total_columns": 4, + "quality_issues": ["age has 10% missing values"], + } + context.heuristic_baseline = Baseline( + name="most_frequent", + model_type="heuristic", + performance=0.6, + metadata={"strategy": "most_frequent"}, + ) + context.excluded_columns = [{"column": "leaky_feature", "reason": "data leakage"}] + return context + + +def _make_evaluation_report() -> EvaluationReport: + core_metrics = CoreMetricsReport( + task_type="binary_classification", + primary_metric_name="roc_auc", + primary_metric_value=0.85, + primary_metric_ci_lower=0.8, + primary_metric_ci_upper=0.9, + all_metrics={"roc_auc": 0.85, "accuracy": 0.8, "brier_score": 0.12}, + metric_confidence_intervals=None, + statistical_notes="Solid performance", + visualizations=None, + ) + diagnostics = DiagnosticReport( + worst_predictions=[], + error_patterns=["Errors concentrated on low tenure"], + subgroup_analysis=None, + key_insights=["Misclassifications skew toward new customers"], + error_distribution_summary="Errors cluster around tenure < 6 months", + ) + robustness = RobustnessReport( + perturbation_tests={"noise": {"impact": "low"}}, + consistency_score=0.92, + robustness_grade="B", + concerns=["Sensitive to rare categories"], + recommendations=["Collect more rare-category samples"], + ) + explainability = ExplainabilityReport( + feature_importance={"age": 0.5, "tenure": 0.3, "balance": 0.2}, + method_used="shap", + top_features=["age", "tenure", "balance"], + confidence_intervals=None, + interpretation="Age and tenure drive predictions", + ) + baseline_comparison = BaselineComparisonReport( + baseline_name="heuristic", + baseline_type="heuristic", + baseline_description="Most frequent class", + baseline_performance={"roc_auc": 0.6}, + model_performance={"roc_auc": 0.85}, + performance_delta={"roc_auc": 0.25}, + performance_delta_pct={"roc_auc": 41.67}, + interpretation="Model outperforms baseline", + ) + + return EvaluationReport( + verdict="PASS", + summary="Model meets quality bar", + deployment_ready=True, + key_concerns=["Monitor drift"], + core_metrics=core_metrics, + diagnostics=diagnostics, + robustness=robustness, + explainability=explainability, + baseline_comparison=baseline_comparison, + recommendations=[{"priority": "HIGH", "action": "Monitor drift", "rationale": "Data shift risk"}], + ) + + +def test_generate_model_card_full_context(tmp_path: Path) -> None: + _write_package_files(tmp_path) + context = _make_full_context(tmp_path) + evaluation_report = _make_evaluation_report() + + final_metrics = {"metric": "roc_auc", "performance": 0.85, "test_samples": 200, "all_metrics": {}} + + model_card = generate_model_card(context, final_metrics, evaluation_report) + + expected_headers = [ + "# Model Card", + "## Summary", + "## Dataset", + "## Features Used", + "## Excluded Columns", + "## Performance", + "## Evaluation Verdict", + "## Hyperparameters", + "## Known Limitations", + "## Reproducibility", + ] + + for header in expected_headers: + assert header in model_card + + assert "`age`" in model_card + assert "max_depth" in model_card + assert "leaky_feature" in model_card + + +def test_generate_model_card_minimal_context(tmp_path: Path) -> None: + context = BuildContext( + user_id="user_min", + experiment_id="exp_min", + dataset_uri="s3://bucket/min.parquet", + work_dir=tmp_path, + intent="Predict outcomes", + ) + + final_metrics = {"metric": "accuracy", "performance": 0.5} + + model_card = generate_model_card(context, final_metrics, evaluation_report=None) + + expected_headers = [ + "# Model Card", + "## Summary", + "## Dataset", + "## Features Used", + "## Performance", + "## Evaluation Verdict", + "## Hyperparameters", + "## Known Limitations", + "## Reproducibility", + ] + + for header in expected_headers: + assert header in model_card + + assert "## Excluded Columns" not in model_card + assert "Not available" in model_card From 513a0480e0678b263dee6c4facf9496260b31116 Mon Sep 17 00:00:00 2001 From: marcellodebernardi Date: Fri, 27 Feb 2026 15:11:28 +0000 Subject: [PATCH 2/3] fix: align model card metadata with plexe naming --- plexe/CODE_INDEX.md | 2 +- .../packaging/model_card_template.py | 47 +++++++++++++++---- tests/CODE_INDEX.md | 2 +- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/plexe/CODE_INDEX.md b/plexe/CODE_INDEX.md index 004a981e..935335bf 100644 --- a/plexe/CODE_INDEX.md +++ b/plexe/CODE_INDEX.md @@ -1,6 +1,6 @@ # Code Index: plexe -> Generated on 2026-02-27 14:59:37 +> Generated on 2026-02-27 15:11:29 Code structure and public interface documentation for the **plexe** package. diff --git a/plexe/templates/packaging/model_card_template.py b/plexe/templates/packaging/model_card_template.py index 3486dbed..7a660912 100644 --- a/plexe/templates/packaging/model_card_template.py +++ b/plexe/templates/packaging/model_card_template.py @@ -189,11 +189,11 @@ def generate_model_card(context, final_metrics: dict, evaluation_report: Any | N metadata = model_metadata.get("metadata", {}) training_timestamp = metadata.get("created_at") or DEFAULT_NOT_AVAILABLE experiment_id = metadata.get("experiment_id") or context.experiment_id or DEFAULT_NOT_AVAILABLE - builder_version = _resolve_model_builder_version() or metadata.get("version") or DEFAULT_NOT_AVAILABLE + plexe_version = _resolve_plexe_version() or metadata.get("version") or DEFAULT_NOT_AVAILABLE lines.append(f"- Experiment ID: {experiment_id}") lines.append(f"- Training timestamp: {training_timestamp}") - lines.append(f"- Model-builder-v2 version: {builder_version}") + lines.append(f"- Plexe version: {plexe_version}") return "\n".join(lines).strip() + "\n" @@ -208,7 +208,8 @@ def _safe_load_json(path: Path) -> dict | None: return None try: with open(path, encoding="utf-8") as f: - return json.load(f) + loaded = json.load(f) + return loaded if isinstance(loaded, dict) else None except (json.JSONDecodeError, OSError, UnicodeDecodeError): return None @@ -218,7 +219,8 @@ def _safe_load_yaml(path: Path) -> dict | None: return None try: with open(path, encoding="utf-8") as f: - return yaml.safe_load(f) + loaded = yaml.safe_load(f) + return loaded if isinstance(loaded, dict) else None except (yaml.YAMLError, OSError, UnicodeDecodeError): return None @@ -226,26 +228,53 @@ def _safe_load_yaml(path: Path) -> dict | None: def _normalize_evaluation_report(evaluation_report: Any | None, package_dir: Path) -> dict | None: if evaluation_report is None: fallback = _safe_load_json(package_dir / "evaluation" / "reports" / "evaluation.json") - return fallback + return _to_plain_dict(fallback) if isinstance(evaluation_report, dict): - return evaluation_report + return _to_plain_dict(evaluation_report) if hasattr(evaluation_report, "to_dict"): try: - return evaluation_report.to_dict() + return _to_plain_dict(evaluation_report.to_dict()) except Exception: pass if hasattr(evaluation_report, "__dict__"): try: - return dict(evaluation_report.__dict__) + return _to_plain_dict(dict(evaluation_report.__dict__)) except Exception: return None return None +def _to_plain_structure(value: Any) -> Any: + if value is None or isinstance(value, str | bytes | bool | numbers.Real): + return value + if isinstance(value, dict): + return {k: _to_plain_structure(v) for k, v in value.items()} + if isinstance(value, list | tuple): + return [_to_plain_structure(v) for v in value] + if hasattr(value, "to_dict"): + try: + return _to_plain_structure(value.to_dict()) + except Exception: + pass + if hasattr(value, "__dict__"): + try: + return _to_plain_structure(dict(value.__dict__)) + except Exception: + pass + return value + + +def _to_plain_dict(value: Any) -> dict | None: + normalized = _to_plain_structure(value) + if isinstance(normalized, dict): + return normalized + return None + + def _get_nested(data: dict, keys: list[str]) -> Any: current: Any = data for key in keys: @@ -549,7 +578,7 @@ def _is_number(value: Any) -> bool: return isinstance(value, numbers.Real) -def _resolve_model_builder_version() -> str | None: +def _resolve_plexe_version() -> str | None: pyproject_path = _find_pyproject() if not pyproject_path: return None diff --git a/tests/CODE_INDEX.md b/tests/CODE_INDEX.md index f05f2080..928b5463 100644 --- a/tests/CODE_INDEX.md +++ b/tests/CODE_INDEX.md @@ -1,6 +1,6 @@ # Code Index: tests -> Generated on 2026-02-27 14:59:37 +> Generated on 2026-02-27 15:11:29 Test suite structure and test case documentation. From 7109c9a0655dcd8b42d60235b4fdb0ca589615e8 Mon Sep 17 00:00:00 2001 From: marcellodebernardi Date: Fri, 27 Feb 2026 15:37:20 +0000 Subject: [PATCH 3/3] chore: bump version to 1.3.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e4d53f83..945d6b1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "plexe" -version = "1.3.2" +version = "1.3.3" description = "An agentic framework for building ML models from natural language" authors = [ "Marcello De Bernardi ",