From c74001f4ac68ccab57813339a774c853ba2a1b3b Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 11:09:26 +0800 Subject: [PATCH 01/40] Update model monitor version --- .../data_drift_compute_metrics/spec.yaml | 2 +- .../data_drift_signal_monitor/spec.yaml | 18 +++++++++--------- .../data_quality_compute_metrics/spec.yaml | 2 +- .../data_quality_metrics_joiner/spec.yaml | 2 +- .../data_quality_signal_monitor/spec.yaml | 18 +++++++++--------- .../data_quality_statistics/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 12 ++++++------ .../feature_importance_metrics/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 6 +++--- .../annotation_compute_histogram/spec.yaml | 2 +- .../annotation_compute_metrics/spec.yaml | 2 +- .../spec.yaml | 12 ++++++------ .../input_schema_adaptor/spec.yaml | 2 +- .../genai_mdc_preprocessor/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_monitor_compute_histogram/spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_monitor_create_manifest/spec.yaml | 2 +- .../model_monitor_data_joiner/spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_monitor_feature_selector/spec.yaml | 2 +- .../model_monitor_metric_outputter/spec.yaml | 2 +- .../model_monitor_output_metrics/spec.yaml | 2 +- .../action_analyzer_correlation_test/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 2 +- .../action_analyzer_output_actions/spec.yaml | 2 +- .../model_monitor_action_analyzer/spec.yaml | 10 +++++----- .../model_monitor_action_detector/spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_performance_signal_monitor/spec.yaml | 8 ++++---- .../prediction_drift_signal_monitor/spec.yaml | 16 ++++++++-------- .../token_statistics_compute_metrics/spec.yaml | 2 +- 36 files changed, 78 insertions(+), 78 deletions(-) diff --git a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml index 21b982d43c..8045c5425b 100644 --- a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_drift_compute_metrics display_name: Data Drift - Compute Metrics description: Compute data drift metrics given a baseline and a deployment's model data input. -version: 0.3.33 +version: 0.3.34 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml index 3e09f8b0c6..c89728fea6 100644 --- a/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: data_drift_signal_monitor display_name: Data Drift - Signal Monitor description: Computes the data drift between a baseline and production data assets. -version: 0.3.54 +version: 0.3.55 is_deterministic: true inputs: @@ -63,7 +63,7 @@ outputs: jobs: compute_feature_importances: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -103,7 +103,7 @@ jobs: type: aml_token compute_drift_metrics: type: spark - component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.33 + component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.34 inputs: production_dataset: type: mltable @@ -130,7 +130,7 @@ jobs: type: aml_token compute_histogram_buckets: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.28 inputs: input_data_1: type: mltable @@ -150,7 +150,7 @@ jobs: type: aml_token compute_baseline_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -170,7 +170,7 @@ jobs: type: aml_token compute_target_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -190,7 +190,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -218,7 +218,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml index c01f1ce5b0..cc67916012 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_compute_metrics display_name: Data Quality - Compute Metrics description: Compute data quality metrics leveraged by the data quality monitor. -version: 0.3.32 +version: 0.3.33 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml index 52850a9483..9e38ca1118 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_metrics_joiner display_name: Data Quality - Metrics Joiner description: Join baseline and target data quality metrics into a single output. -version: 0.3.25 +version: 0.3.26 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml index 0f0bfce699..2e0295d8a1 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: data_quality_signal_monitor display_name: Data Quality - Signal Monitor description: Computes the data quality of a target dataset with reference to a baseline. -version: 0.3.52 +version: 0.3.53 is_deterministic: true inputs: @@ -63,7 +63,7 @@ outputs: jobs: compute_feature_importances: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -103,7 +103,7 @@ jobs: type: aml_token compute_baseline_data_statistics: type: spark - component: azureml://registries/azureml/components/data_quality_data_statistics/versions/0.3.27 + component: azureml://registries/azureml/components/data_quality_data_statistics/versions/0.3.28 inputs: baseline_data: type: mltable @@ -120,7 +120,7 @@ jobs: type: aml_token compute_baseline_data_quality: type: spark - component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.33 inputs: input_data: type: mltable @@ -145,7 +145,7 @@ jobs: type: aml_token compute_target_data_quality: type: spark - component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.33 inputs: input_data: type: mltable @@ -168,7 +168,7 @@ jobs: type: aml_token join_data_quality_metrics: type: spark - component: azureml://registries/azureml/components/data_quality_metrics_joiner/versions/0.3.25 + component: azureml://registries/azureml/components/data_quality_metrics_joiner/versions/0.3.26 inputs: baseline_metrics: type: mltable @@ -188,7 +188,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -210,7 +210,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml index e5e38dcbe2..5082770bb7 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_data_statistics display_name: Data Quality - Data Statistics description: Compute data statistics leveraged by the data quality monitor. -version: 0.3.27 +version: 0.3.28 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml index a4a5b7d4ab..4501cf2d82 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml @@ -2,7 +2,7 @@ $schema: http://azureml/sdk-2-0/SparkComponent.json type: spark name: feature_attribution_drift_compute_metrics -version: 0.3.28 +version: 0.3.29 display_name: Feature Attribution Drift - Compute Metrics is_deterministic: true description: Feature attribution drift using model monitoring. diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml index 8e47e34117..e3ad247fdc 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: feature_attribution_drift_signal_monitor display_name: Feature Attribution Drift - Signal Monitor description: Computes the feature attribution between a baseline and production data assets. -version: 0.3.44 +version: 0.3.45 is_deterministic: true inputs: @@ -44,7 +44,7 @@ outputs: jobs: compute_baseline_explanations: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -63,7 +63,7 @@ jobs: type: aml_token compute_production_explanations: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token compute_feature_attribution: type: spark - component: azureml://registries/azureml/components/feature_attribution_drift_compute_metrics/versions/0.3.28 + component: azureml://registries/azureml/components/feature_attribution_drift_compute_metrics/versions/0.3.29 inputs: production_data: type: mltable @@ -101,7 +101,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -122,7 +122,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml index 661f7990c4..f251573d39 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml @@ -2,7 +2,7 @@ $schema: http://azureml/sdk-2-0/SparkComponent.json type: spark name: feature_importance_metrics -version: 0.3.31 +version: 0.3.32 display_name: Feature importance is_deterministic: true description: Feature importance for model monitoring. diff --git a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml index a3b5812c2f..7716d3d56e 100644 --- a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: genai_token_statistics_compute_metrics display_name: GenAI Token Statistics - Compute Metrics description: Compute token statistics metrics. -version: 0.0.21 +version: 0.0.22 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml index 9a88b12c12..fa775d2a86 100644 --- a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: genai_token_statistics_signal_monitor display_name: GenAI Token Statistics - Signal Monitor description: Computes the token and cost metrics over LLM outputs. -version: 0.0.21 +version: 0.0.22 is_deterministic: true inputs: monitor_name: @@ -30,7 +30,7 @@ outputs: jobs: compute_metrics: type: spark - component: azureml://registries/azureml/components/genai_token_statistics_compute_metrics/versions/0.0.21 + component: azureml://registries/azureml/components/genai_token_statistics_compute_metrics/versions/0.0.22 inputs: production_dataset: type: mltable @@ -49,7 +49,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index d7d7442008..b5570b7957 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_annotation_compute_histogram display_name: Annotation - Compute Histogram description: Compute annotation histogram given a deployment's model data input. -version: 0.4.36 +version: 0.4.37 is_deterministic: false inputs: production_dataset: diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml index 5f52d05793..a809425699 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_annotation_compute_metrics display_name: Annotation - Compute Metrics description: Compute annotation metrics given a deployment's model data input. -version: 0.4.29 +version: 0.4.30 is_deterministic: True inputs: annotation_histogram: diff --git a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml index cf9c1afc9d..de338175a4 100644 --- a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: generation_safety_quality_signal_monitor display_name: Generation Safety & Quality - Signal Monitor description: Computes the content generation safety metrics over LLM outputs. -version: 0.5.27 +version: 0.5.28 is_deterministic: true inputs: monitor_name: @@ -109,7 +109,7 @@ outputs: jobs: input_schema_adaptor: type: spark - component: azureml://registries/azureml/components/gsq_input_schema_adaptor/versions/0.0.24 + component: azureml://registries/azureml/components/gsq_input_schema_adaptor/versions/0.0.25 inputs: production_dataset: type: mltable @@ -124,7 +124,7 @@ jobs: type: aml_token compute_histogram: type: spark - component: azureml://registries/azureml/components/gsq_annotation_compute_histogram/versions/0.4.36 + component: azureml://registries/azureml/components/gsq_annotation_compute_histogram/versions/0.4.37 inputs: production_dataset: type: mltable @@ -167,7 +167,7 @@ jobs: type: aml_token compute_metrics: type: spark - component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.29 + component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.30 inputs: annotation_histogram: type: mltable @@ -189,7 +189,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable @@ -211,7 +211,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml index 60cbab7f0e..cc6af8ff08 100644 --- a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_input_schema_adaptor display_name: Input Schema Adaptor description: Adapt data to fit into GSQ component. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: production_dataset: diff --git a/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml index 07a80d385a..c1f8dfa494 100644 --- a/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: genai_mdc_preprocessor display_name: GenAI MDC - Preprocessor description: Filters the raw span log based on the window provided, and aggregates it to trace level. -version: 0.0.28 +version: 0.0.29 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml index 95c86b4da3..5074803ac2 100644 --- a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_data_collector_preprocessor display_name: Model Data Collector - Preprocessor description: Filters the data based on the window provided. -version: 0.4.29 +version: 0.4.30 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml index 96358f208a..aa7006fa0e 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_azmon_metric_publisher display_name: Model Monitor - Azure Monitor Metric Publisher description: Azure Monitor Publisher for the computed model monitor metrics. -version: 0.3.31 +version: 0.3.32 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml index bea674b797..40946e57a9 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_compute_histogram display_name: Model Monitor - Compute Histogram description: Compute a histogram given an input data and associated histogram buckets. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml index f1d18eb705..926d631c58 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_compute_histogram_buckets display_name: Model Monitor - Compute Histogram Buckets description: Compute histogram buckets given up to two datasets. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml index ba25c5f5cb..e5a41373c7 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_create_manifest display_name: Model Monitor - Create Manifest description: Creates the model monitor metric manifest. -version: 0.3.24 +version: 0.3.25 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml index 0c07c9735b..9b322230e2 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_data_joiner display_name: Model Monitor - Data Joiner description: Joins two data assets on the given columns for model monitor. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml index 713342d63e..8d124db6f2 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_evaluate_metrics_threshold display_name: Model Monitor - Evaluate Metrics Threshold description: Evaluate signal metrics against the threshold provided in the monitoring signal. -version: 0.3.32 +version: 0.3.33 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml index e24e594285..30c18ac735 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_feature_selector display_name: Model Monitor - Feature Selector description: Selects features to compute signal metrics on. -version: 0.3.28 +version: 0.3.29 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml index 722dcfe219..83f172e9cc 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_metric_outputter display_name: Model Monitor - Metric Outputter description: Output the computed model monitor metrics. -version: 0.3.35 +version: 0.3.36 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml index c1e828fbbe..1c1507eb1f 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_output_metrics display_name: Model Monitor - Output Metrics description: Output the computed model monitor metrics to the default datastore. -version: 0.3.32 +version: 0.3.33 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml index 8c8b8f84de..0c9dcae963 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_correlation_test display_name: Action Analyzer - Correlation Test description: Perform correlation test on different groups to generate actions. -version: 0.0.21 +version: 0.0.22 is_deterministic: True inputs: data_with_action_metric_score: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml index 0b9b97cd39..67b5bdfada 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_identify_problem_traffic display_name: Action Analyzer - Identify Problem Traffic description: Separate bad queries into different groups. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: signal_output: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml index 658714a7fc..f054b6b3ae 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_metrics_calculation display_name: Action Analyzer - Metrics Calculation description: Calculate futher metrics for generating actions. -version: 0.0.21 +version: 0.0.22 is_deterministic: True inputs: data_with_groups: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml index 7bd80a94ab..6c15cddc1e 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_output_actions display_name: Action Analyzer - Output Actions description: Merge and output actions. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: action_data: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml index 42d2d8bb59..42e8f567eb 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: model_monitor_action_analyzer display_name: Model Monitor - Action Analyzer description: Generate and output actions to the default datastore. -version: 0.0.25 +version: 0.0.26 is_deterministic: true inputs: signal_output: @@ -52,7 +52,7 @@ outputs: jobs: identify_problem_traffic: type: spark - component: azureml://registries/azureml/components/action_analyzer_identify_problem_traffic/versions/0.0.24 + component: azureml://registries/azureml/components/action_analyzer_identify_problem_traffic/versions/0.0.25 inputs: signal_output: type: uri_folder @@ -74,7 +74,7 @@ jobs: type: aml_token metrics_calculation: type: spark - component: azureml://registries/azureml/components/action_analyzer_metrics_calculation/versions/0.0.21 + component: azureml://registries/azureml/components/action_analyzer_metrics_calculation/versions/0.0.22 inputs: data_with_groups: type: mltable @@ -91,7 +91,7 @@ jobs: type: aml_token correlation_test: type: spark - component: azureml://registries/azureml/components/action_analyzer_correlation_test/versions/0.0.21 + component: azureml://registries/azureml/components/action_analyzer_correlation_test/versions/0.0.22 inputs: data_with_action_metric_score: type: mltable @@ -106,7 +106,7 @@ jobs: type: aml_token output_actions: type: spark - component: azureml://registries/azureml/components/action_analyzer_output_actions/versions/0.0.24 + component: azureml://registries/azureml/components/action_analyzer_output_actions/versions/0.0.25 inputs: action_data: type: mltable diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml index 467321060c..ec21779dde 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_action_detector display_name: Model Monitor - Action Detector description: Generate and output actions -version: 0.0.17 +version: 0.0.18 is_deterministic: true inputs: signal_output: diff --git a/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml b/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml index 7650b59b1c..8bfe429cff 100644 --- a/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_performance_compute_metrics display_name: Model Performance - Compute Metrics description: Compute model performance metrics leveraged by the model performance monitor. -version: 0.0.26 +version: 0.0.27 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml b/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml index 766d3608f8..da8e745b64 100644 --- a/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: model_performance_signal_monitor display_name: Model Performance - Signal Monitor description: Computes the model performance -version: 0.0.28 +version: 0.0.29 is_deterministic: true inputs: task: @@ -57,7 +57,7 @@ outputs: jobs: compute_metrics: type: spark - component: azureml://registries/azureml/components/model_performance_compute_metrics/versions/0.0.26 + component: azureml://registries/azureml/components/model_performance_compute_metrics/versions/0.0.27 inputs: task: ${{parent.inputs.task}} baseline_data_target_column: ${{parent.inputs.baseline_data_target_column}} @@ -78,7 +78,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable @@ -99,7 +99,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml index b2cdafdf3e..094db6d9f4 100644 --- a/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: prediction_drift_signal_monitor display_name: Prediction Drift - Signal Monitor description: Computes the prediction drift between a baseline and a target data assets. -version: 0.4.30 +version: 0.4.31 is_deterministic: true inputs: @@ -57,7 +57,7 @@ outputs: jobs: feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -77,7 +77,7 @@ jobs: type: aml_token compute_drift_metrics: type: spark - component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.33 + component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.34 inputs: production_dataset: type: mltable @@ -104,7 +104,7 @@ jobs: type: aml_token compute_histogram_buckets: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.28 inputs: input_data_1: type: mltable @@ -124,7 +124,7 @@ jobs: type: aml_token compute_baseline_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -144,7 +144,7 @@ jobs: type: aml_token compute_target_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -164,7 +164,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -191,7 +191,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml b/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml index 07c65e989c..338c5322cd 100644 --- a/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: token_statistics_compute_metrics display_name: Token Statistics - Compute Metrics description: Compute token statistics metrics. -version: 0.0.23 +version: 0.0.24 is_deterministic: true inputs: From 3670f2ca8a1325fc9c9a0fc3b2d8c0ab3d5a3739 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 14:24:19 +0800 Subject: [PATCH 02/40] fix --- .../tests/e2e/test_generation_safety_quality_e2e.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index afd190fecc..32bd9c9ba1 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -96,7 +96,8 @@ def test_generation_safety_quality_successful( } ) - assert pipeline_job.status == "Completed" + job_details = ml_client.jobs.get(pipeline_job.name) + assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={getattr(job_details, 'error', None)}" def test_generation_safety_quality_genai_successful( self, ml_client: MLClient, get_component, submit_pipeline_job, test_suite_name @@ -115,4 +116,5 @@ def test_generation_safety_quality_genai_successful( } ) - assert pipeline_job.status == "Completed" + job_details = ml_client.jobs.get(pipeline_job.name) + assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={getattr(job_details, 'error', None)}" From 5d012bbac39ca40bcda1ce5204894c91ae4be8fd Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 19:45:52 +0800 Subject: [PATCH 03/40] fix --- .../components/tests/e2e/test_generation_safety_quality_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 32bd9c9ba1..44e641a6ff 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -97,7 +97,7 @@ def test_generation_safety_quality_successful( ) job_details = ml_client.jobs.get(pipeline_job.name) - assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={getattr(job_details, 'error', None)}" + assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={job_details}" def test_generation_safety_quality_genai_successful( self, ml_client: MLClient, get_component, submit_pipeline_job, test_suite_name From 658340745b8777efb562037984b15805ea920c21 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 19:53:02 +0800 Subject: [PATCH 04/40] fix --- .../components/tests/e2e/test_generation_safety_quality_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 44e641a6ff..4ad540d048 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -117,4 +117,4 @@ def test_generation_safety_quality_genai_successful( ) job_details = ml_client.jobs.get(pipeline_job.name) - assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={getattr(job_details, 'error', None)}" + assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={job_details}" From 3dd6fd8cb87e06d107501d1f4d2d23a44384f8a7 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 22:05:07 +0800 Subject: [PATCH 05/40] fix --- .../tests/e2e/test_generation_safety_quality_e2e.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 4ad540d048..9dd80bc8fb 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -97,6 +97,11 @@ def test_generation_safety_quality_successful( ) job_details = ml_client.jobs.get(pipeline_job.name) + if pipeline_job.status != "Completed": + from azure.ai.ml import MLClient + job = ml_client.jobs.get(pipeline_job.name) + print(job) + assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={job_details}" def test_generation_safety_quality_genai_successful( @@ -117,4 +122,9 @@ def test_generation_safety_quality_genai_successful( ) job_details = ml_client.jobs.get(pipeline_job.name) + if pipeline_job.status != "Completed": + from azure.ai.ml import MLClient + job = ml_client.jobs.get(pipeline_job.name) + print(job) + assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={job_details}" From 7a6d4963fd5b13d3d397b73380959e93e6b23552 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 23:38:02 +0800 Subject: [PATCH 06/40] fix --- .../e2e/test_generation_safety_quality_e2e.py | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 9dd80bc8fb..d16f2e3683 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. """This file contains e2e tests for the data drift model monitor component.""" - +import os import pytest from azure.ai.ml import MLClient, Output from azure.ai.ml.dsl import pipeline @@ -75,6 +75,28 @@ def _generation_safety_quality_signal_monitor_e2e(): return ml_client.jobs.get(pipeline_job.name) +def download_step_logs(ml_client, pipeline_job_name, step_name, download_dir="./logs"): + """ + 查找指定 pipeline job 下的 step，并下载日志到本地。 + """ + found = False + for child in ml_client.jobs.list(parent_job_name=pipeline_job_name): + # 有的版本 display_name 可能是 step name，也可能是别名，视你的 pipeline 定义 + if child.display_name == step_name or child.name.endswith(step_name): + found = True + print(f"Found step: {child.display_name} ({child.name})，开始下载日志...") + ml_client.jobs.download(child.name, download_path=download_dir, all=True) + print(f"日志已下载到 {os.path.abspath(download_dir)}") + # 可选：打印部分日志内容 + log_file = os.path.join(download_dir, child.name, 'logs', '70_driver_log.txt') + if os.path.exists(log_file): + with open(log_file, 'r', encoding='utf-8') as f: + print("----- 70_driver_log.txt 部分内容 -----") + print(f.read(2000)) # 打印前2000字符 + break + if not found: + print(f"未找到 step: {step_name}，请确认名称是否正确。") + @pytest.mark.gsq_test @pytest.mark.e2e class TestGenerationSafetyQualityModelMonitor: @@ -98,11 +120,16 @@ def test_generation_safety_quality_successful( job_details = ml_client.jobs.get(pipeline_job.name) if pipeline_job.status != "Completed": - from azure.ai.ml import MLClient - job = ml_client.jobs.get(pipeline_job.name) - print(job) - - assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={job_details}" + print(job_details) + # 自动下载 logs + download_step_logs( + ml_client, + pipeline_job.name, + "generation_safety_quality_signal_monitor_output", + download_dir="./logs" + ) + + assert pipeline_job.status == "Completed" def test_generation_safety_quality_genai_successful( self, ml_client: MLClient, get_component, submit_pipeline_job, test_suite_name @@ -123,8 +150,13 @@ def test_generation_safety_quality_genai_successful( job_details = ml_client.jobs.get(pipeline_job.name) if pipeline_job.status != "Completed": - from azure.ai.ml import MLClient - job = ml_client.jobs.get(pipeline_job.name) - print(job) - - assert pipeline_job.status == "Completed", f"Job failed! status={pipeline_job.status}, error={job_details}" + print(job_details) + # 自动下载 logs + download_step_logs( + ml_client, + pipeline_job.name, + "generation_safety_quality_signal_monitor_output", + download_dir="./logs" + ) + + assert pipeline_job.status == "Completed" \ No newline at end of file From a75c6ea8c54abd8cf69421e462c4f70a4d65c0d9 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 01:17:37 +0800 Subject: [PATCH 07/40] fix --- .../e2e/test_generation_safety_quality_e2e.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index d16f2e3683..8597c979d6 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -77,26 +77,40 @@ def _generation_safety_quality_signal_monitor_e2e(): def download_step_logs(ml_client, pipeline_job_name, step_name, download_dir="./logs"): """ - 查找指定 pipeline job 下的 step，并下载日志到本地。 + 查找指定 pipeline job 下的 step，并下载日志到本地，并列出所有日志文件。 """ found = False for child in ml_client.jobs.list(parent_job_name=pipeline_job_name): - # 有的版本 display_name 可能是 step name，也可能是别名，视你的 pipeline 定义 if child.display_name == step_name or child.name.endswith(step_name): found = True print(f"Found step: {child.display_name} ({child.name})，开始下载日志...") ml_client.jobs.download(child.name, download_path=download_dir, all=True) - print(f"日志已下载到 {os.path.abspath(download_dir)}") - # 可选：打印部分日志内容 - log_file = os.path.join(download_dir, child.name, 'logs', '70_driver_log.txt') - if os.path.exists(log_file): - with open(log_file, 'r', encoding='utf-8') as f: - print("----- 70_driver_log.txt 部分内容 -----") - print(f.read(2000)) # 打印前2000字符 + step_dir = os.path.join(download_dir, child.name) + print(f"日志已下载到 {os.path.abspath(step_dir)}") + + # 列出 logs 目录下的所有文件 + logs_dir = os.path.join(step_dir, 'logs') + if os.path.exists(logs_dir): + print("\n----- logs 目录下的所有日志文件 -----") + for root, dirs, files in os.walk(logs_dir): + for file in files: + log_path = os.path.join(root, file) + rel_path = os.path.relpath(log_path, start=logs_dir) + print(rel_path) + # 可选：打印每个日志文件的前200字符 + try: + with open(log_path, 'r', encoding='utf-8') as f: + print(f"--- {rel_path} 内容预览 ---") + print(f.read(2000)) + print("文件完毕\n\n\n\n") + except Exception as e: + print(f"无法读取 {rel_path}: {e}") + else: + print(f"未找到 logs 目录: {logs_dir}") break if not found: print(f"未找到 step: {step_name}，请确认名称是否正确。") - + @pytest.mark.gsq_test @pytest.mark.e2e class TestGenerationSafetyQualityModelMonitor: From bb8bf62538e473679c832ca598f275940d716504 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 11:23:28 +0800 Subject: [PATCH 08/40] fix --- .../e2e/test_generation_safety_quality_e2e.py | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 8597c979d6..daecd2cd70 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -75,38 +75,50 @@ def _generation_safety_quality_signal_monitor_e2e(): return ml_client.jobs.get(pipeline_job.name) -def download_step_logs(ml_client, pipeline_job_name, step_name, download_dir="./logs"): - """ - 查找指定 pipeline job 下的 step，并下载日志到本地，并列出所有日志文件。 - """ +def download_and_flatten_logs(ml_client, pipeline_job_name, step_name, download_dir="./log"): found = False for child in ml_client.jobs.list(parent_job_name=pipeline_job_name): - if child.display_name == step_name or child.name.endswith(step_name): + if child.display_name == step_name: found = True print(f"Found step: {child.display_name} ({child.name})，开始下载日志...") - ml_client.jobs.download(child.name, download_path=download_dir, all=True) - step_dir = os.path.join(download_dir, child.name) - print(f"日志已下载到 {os.path.abspath(step_dir)}") + # 下载到临时目录 + temp_dir = os.path.join(download_dir, "temp_" + child.name) + ml_client.jobs.download(child.name, download_path=temp_dir, all=True) + print(f"日志已下载到 {os.path.abspath(temp_dir)}") - # 列出 logs 目录下的所有文件 - logs_dir = os.path.join(step_dir, 'logs') - if os.path.exists(logs_dir): - print("\n----- logs 目录下的所有日志文件 -----") - for root, dirs, files in os.walk(logs_dir): - for file in files: - log_path = os.path.join(root, file) - rel_path = os.path.relpath(log_path, start=logs_dir) - print(rel_path) - # 可选：打印每个日志文件的前200字符 - try: - with open(log_path, 'r', encoding='utf-8') as f: - print(f"--- {rel_path} 内容预览 ---") - print(f.read(2000)) - print("文件完毕\n\n\n\n") - except Exception as e: - print(f"无法读取 {rel_path}: {e}") - else: - print(f"未找到 logs 目录: {logs_dir}") + # 创建目标 log 目录 + os.makedirs(download_dir, exist_ok=True) + count = 0 + + # 遍历 temp_dir 下所有文件，复制到 download_dir 下（文件名加上 step/job 前缀防止重名） + for root, dirs, files in os.walk(temp_dir): + for file in files: + src_path = os.path.join(root, file) + # 生成新文件名，防止同名覆盖 + rel_path = os.path.relpath(src_path, start=temp_dir) + new_file_name = f"{child.name.replace('/', '_')}_{rel_path.replace('/', '_')}" + dst_path = os.path.join(download_dir, new_file_name) + shutil.copy2(src_path, dst_path) + count += 1 + print(f"已整理 {count} 个日志文件到 {os.path.abspath(download_dir)}") + + # 删除临时目录 + shutil.rmtree(temp_dir) + + # 列出所有日志文件，并预览内容 + print("\n----- log 目录下的所有日志文件 -----") + for file in os.listdir(download_dir): + file_path = os.path.join(download_dir, file) + if os.path.isfile(file_path): + print(file) + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read(2000) + print(f"--- {file} 内容预览 ---") + print(content if content else "(文件为空)") + print("文件完毕\n\n\n\n") + except Exception as e: + print(f"无法读取 {file}: {e}") break if not found: print(f"未找到 step: {step_name}，请确认名称是否正确。") @@ -136,7 +148,7 @@ def test_generation_safety_quality_successful( if pipeline_job.status != "Completed": print(job_details) # 自动下载 logs - download_step_logs( + download_and_flatten_logs( ml_client, pipeline_job.name, "generation_safety_quality_signal_monitor_output", From 719194f91ba64c7863ea3f88484f010ae3b05147 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 12:36:04 +0800 Subject: [PATCH 09/40] fix --- .../components/tests/e2e/test_generation_safety_quality_e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index daecd2cd70..59934a407e 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -3,6 +3,7 @@ """This file contains e2e tests for the data drift model monitor component.""" import os +import shutil import pytest from azure.ai.ml import MLClient, Output from azure.ai.ml.dsl import pipeline @@ -178,7 +179,7 @@ def test_generation_safety_quality_genai_successful( if pipeline_job.status != "Completed": print(job_details) # 自动下载 logs - download_step_logs( + download_and_flatten_logs( ml_client, pipeline_job.name, "generation_safety_quality_signal_monitor_output", From 72f390a31b6b99b7171843ace668fed7d148ec4d Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 15:03:23 +0800 Subject: [PATCH 10/40] fix --- .../components/tests/e2e/test_generation_safety_quality_e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 59934a407e..84df308aca 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -79,7 +79,8 @@ def _generation_safety_quality_signal_monitor_e2e(): def download_and_flatten_logs(ml_client, pipeline_job_name, step_name, download_dir="./log"): found = False for child in ml_client.jobs.list(parent_job_name=pipeline_job_name): - if child.display_name == step_name: + print(f"Found child job: {child.display_name} ({child.name})") + # if child.display_name == step_name: found = True print(f"Found step: {child.display_name} ({child.name})，开始下载日志...") # 下载到临时目录 From 2d0fad3024ef2949f5a0f3412f044eebe145d920 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 17:54:24 +0800 Subject: [PATCH 11/40] fix --- .../e2e/test_generation_safety_quality_e2e.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 84df308aca..37b7449adc 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -146,16 +146,20 @@ def test_generation_safety_quality_successful( } ) + # job_details = ml_client.jobs.get(pipeline_job.name) + # if pipeline_job.status != "Completed": + # print(job_details) + # # 自动下载 logs + # download_and_flatten_logs( + # ml_client, + # pipeline_job.name, + # "generation_safety_quality_signal_monitor_output", + # download_dir="./logs" + # ) job_details = ml_client.jobs.get(pipeline_job.name) - if pipeline_job.status != "Completed": - print(job_details) - # 自动下载 logs - download_and_flatten_logs( - ml_client, - pipeline_job.name, - "generation_safety_quality_signal_monitor_output", - download_dir="./logs" - ) + if hasattr(job_details, "error") and job_details.error: + print("AzureML Job 错误详情：") + print(job_details.error) assert pipeline_job.status == "Completed" @@ -176,15 +180,15 @@ def test_generation_safety_quality_genai_successful( } ) - job_details = ml_client.jobs.get(pipeline_job.name) - if pipeline_job.status != "Completed": - print(job_details) - # 自动下载 logs - download_and_flatten_logs( - ml_client, - pipeline_job.name, - "generation_safety_quality_signal_monitor_output", - download_dir="./logs" - ) - - assert pipeline_job.status == "Completed" \ No newline at end of file + # job_details = ml_client.jobs.get(pipeline_job.name) + # if pipeline_job.status != "Completed": + # print(job_details) + # # 自动下载 logs + # download_and_flatten_logs( + # ml_client, + # pipeline_job.name, + # "generation_safety_quality_signal_monitor_output", + # download_dir="./logs" + # ) + + # assert pipeline_job.status == "Completed" \ No newline at end of file From 977e792135586febcf73390465871cba73c3758b Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 19:16:32 +0800 Subject: [PATCH 12/40] fix --- .../tests/e2e/test_generation_safety_quality_e2e.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 37b7449adc..b53301fe64 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -156,10 +156,11 @@ def test_generation_safety_quality_successful( # "generation_safety_quality_signal_monitor_output", # download_dir="./logs" # ) - job_details = ml_client.jobs.get(pipeline_job.name) - if hasattr(job_details, "error") and job_details.error: + + if pipeline_job.status != "Completed": + job_details = ml_client.jobs.get(pipeline_job.name) print("AzureML Job 错误详情：") - print(job_details.error) + print(job_details) assert pipeline_job.status == "Completed" From 21f0e7db49f9b856fb09b54e29da0409c6e1f8b0 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 7 May 2025 22:06:07 +0800 Subject: [PATCH 13/40] fix --- .../components/tests/e2e/test_generation_safety_quality_e2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index b53301fe64..8d840db567 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -160,7 +160,8 @@ def test_generation_safety_quality_successful( if pipeline_job.status != "Completed": job_details = ml_client.jobs.get(pipeline_job.name) print("AzureML Job 错误详情：") - print(job_details) + print("Job status:", job_details.status) + print("Job error:", job_details.error) assert pipeline_job.status == "Completed" From a6054a6da6d19f85ac97c063b607a5d9d925351e Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 00:07:55 +0800 Subject: [PATCH 14/40] fix --- .../e2e/test_generation_safety_quality_e2e.py | 82 ++----------------- 1 file changed, 8 insertions(+), 74 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 8d840db567..4f7132d541 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -75,56 +75,6 @@ def _generation_safety_quality_signal_monitor_e2e(): return ml_client.jobs.get(pipeline_job.name) - -def download_and_flatten_logs(ml_client, pipeline_job_name, step_name, download_dir="./log"): - found = False - for child in ml_client.jobs.list(parent_job_name=pipeline_job_name): - print(f"Found child job: {child.display_name} ({child.name})") - # if child.display_name == step_name: - found = True - print(f"Found step: {child.display_name} ({child.name})，开始下载日志...") - # 下载到临时目录 - temp_dir = os.path.join(download_dir, "temp_" + child.name) - ml_client.jobs.download(child.name, download_path=temp_dir, all=True) - print(f"日志已下载到 {os.path.abspath(temp_dir)}") - - # 创建目标 log 目录 - os.makedirs(download_dir, exist_ok=True) - count = 0 - - # 遍历 temp_dir 下所有文件，复制到 download_dir 下（文件名加上 step/job 前缀防止重名） - for root, dirs, files in os.walk(temp_dir): - for file in files: - src_path = os.path.join(root, file) - # 生成新文件名，防止同名覆盖 - rel_path = os.path.relpath(src_path, start=temp_dir) - new_file_name = f"{child.name.replace('/', '_')}_{rel_path.replace('/', '_')}" - dst_path = os.path.join(download_dir, new_file_name) - shutil.copy2(src_path, dst_path) - count += 1 - print(f"已整理 {count} 个日志文件到 {os.path.abspath(download_dir)}") - - # 删除临时目录 - shutil.rmtree(temp_dir) - - # 列出所有日志文件，并预览内容 - print("\n----- log 目录下的所有日志文件 -----") - for file in os.listdir(download_dir): - file_path = os.path.join(download_dir, file) - if os.path.isfile(file_path): - print(file) - try: - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read(2000) - print(f"--- {file} 内容预览 ---") - print(content if content else "(文件为空)") - print("文件完毕\n\n\n\n") - except Exception as e: - print(f"无法读取 {file}: {e}") - break - if not found: - print(f"未找到 step: {step_name}，请确认名称是否正确。") - @pytest.mark.gsq_test @pytest.mark.e2e class TestGenerationSafetyQualityModelMonitor: @@ -146,20 +96,9 @@ def test_generation_safety_quality_successful( } ) - # job_details = ml_client.jobs.get(pipeline_job.name) - # if pipeline_job.status != "Completed": - # print(job_details) - # # 自动下载 logs - # download_and_flatten_logs( - # ml_client, - # pipeline_job.name, - # "generation_safety_quality_signal_monitor_output", - # download_dir="./logs" - # ) - if pipeline_job.status != "Completed": job_details = ml_client.jobs.get(pipeline_job.name) - print("AzureML Job 错误详情：") + print("AzureML Job error detail：") print("Job status:", job_details.status) print("Job error:", job_details.error) @@ -182,15 +121,10 @@ def test_generation_safety_quality_genai_successful( } ) - # job_details = ml_client.jobs.get(pipeline_job.name) - # if pipeline_job.status != "Completed": - # print(job_details) - # # 自动下载 logs - # download_and_flatten_logs( - # ml_client, - # pipeline_job.name, - # "generation_safety_quality_signal_monitor_output", - # download_dir="./logs" - # ) - - # assert pipeline_job.status == "Completed" \ No newline at end of file + if pipeline_job.status != "Completed": + job_details = ml_client.jobs.get(pipeline_job.name) + print("AzureML Job error detail：") + print("Job status:", job_details.status) + print("Job error:", job_details.error) + + assert pipeline_job.status == "Completed" From e05931cc1cfaa6494045b744d69bbb826df6c4ff Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 01:31:34 +0800 Subject: [PATCH 15/40] fix --- .../e2e/test_generation_safety_quality_e2e.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 4f7132d541..e370d23d09 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -2,8 +2,7 @@ # Licensed under the MIT License. """This file contains e2e tests for the data drift model monitor component.""" -import os -import shutil + import pytest from azure.ai.ml import MLClient, Output from azure.ai.ml.dsl import pipeline @@ -54,6 +53,7 @@ def _generation_safety_quality_signal_monitor_e2e(): completion_column_name=completion_column_name, ground_truth_column_name=ground_truth_column_name, context_column_name=context_column_name, + instance_type="standard_e8s_v3", ) return { "signal_output": generation_safety_quality_signal_monitor_output.outputs.signal_output @@ -61,7 +61,6 @@ def _generation_safety_quality_signal_monitor_e2e(): pipeline_job = _generation_safety_quality_signal_monitor_e2e() pipeline_job.outputs.signal_output = Output(type="uri_folder", mode="direct") - pipeline_job = submit_pipeline_job( pipeline_job, experiment_name, expect_failure ) @@ -96,12 +95,6 @@ def test_generation_safety_quality_successful( } ) - if pipeline_job.status != "Completed": - job_details = ml_client.jobs.get(pipeline_job.name) - print("AzureML Job error detail：") - print("Job status:", job_details.status) - print("Job error:", job_details.error) - assert pipeline_job.status == "Completed" def test_generation_safety_quality_genai_successful( @@ -121,10 +114,4 @@ def test_generation_safety_quality_genai_successful( } ) - if pipeline_job.status != "Completed": - job_details = ml_client.jobs.get(pipeline_job.name) - print("AzureML Job error detail：") - print("Job status:", job_details.status) - print("Job error:", job_details.error) - assert pipeline_job.status == "Completed" From 4e1658946ed2cff2b220c1410c1b9d0ed8b7b513 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 13:05:46 +0800 Subject: [PATCH 16/40] fix --- .../annotation_compute_histogram/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index b5570b7957..dac995d708 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -102,6 +102,7 @@ conf: spark.jars.packages: com.microsoft.azure:synapseml_2.12:0.11.0-26-52919ce4-SNAPSHOT,org.apache.spark:spark-avro_2.12:3.3.1 spark.jars.repositories: https://mmlspark.azureedge.net/maven spark.jars.excludes: org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind + spark.hadoop.aml.enable_cache: true spark.yarn.user.classpath.first: true spark.sql.parquet.enableVectorizedReader: false spark.sql.legacy.replaceDatabricksSparkAvro.enabled: true From 8a22b1d112f81886f1d1610b884e8e486366a55e Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 15:35:34 +0800 Subject: [PATCH 17/40] fix --- .../components/tests/e2e/test_generation_safety_quality_e2e.py | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index e370d23d09..15f0d78fa1 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -74,6 +74,7 @@ def _generation_safety_quality_signal_monitor_e2e(): return ml_client.jobs.get(pipeline_job.name) + @pytest.mark.gsq_test @pytest.mark.e2e class TestGenerationSafetyQualityModelMonitor: From 2414f52fc9fa941528db33f6865762317005db3f Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 19:08:43 +0800 Subject: [PATCH 18/40] fix --- .../generation_safety_quality_signal_monitor/spec.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml index de338175a4..a3d84b7158 100644 --- a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml @@ -119,7 +119,7 @@ jobs: type: mltable resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.3" + runtime_version: "3.4" identity: type: aml_token compute_histogram: @@ -162,7 +162,7 @@ jobs: path: ${{parent.outputs.signal_scored_data}} resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.3" + runtime_version: "3.4" identity: type: aml_token compute_metrics: @@ -184,7 +184,7 @@ jobs: type: mltable resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.3" + runtime_version: "3.4" identity: type: aml_token output_signal_metrics: @@ -206,7 +206,7 @@ jobs: mode: direct resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.3" + runtime_version: "3.4" identity: type: aml_token evaluate_metric_thresholds: @@ -220,6 +220,6 @@ jobs: signal_name: ${{parent.inputs.signal_name}} resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.3" + runtime_version: "3.4" identity: type: aml_token From f73959a1a010aea72c55dde84286b5617c44caf9 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 19:58:01 +0800 Subject: [PATCH 19/40] fix --- .../generation_safety_quality/input_schema_adaptor/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml index cc6af8ff08..8b76dabbe7 100644 --- a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml @@ -39,6 +39,7 @@ conf: - azureml-fsspec~=1.0.0 - fsspec~=2023.4.0 - numpy<2.0.0 + - cryptography==42.0.5 name: momo-base-spark code: ../../src From c2b0177d684352da9280efea58cf09ba5535a3f5 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 21:04:50 +0800 Subject: [PATCH 20/40] fix --- .../generation_safety_quality/input_schema_adaptor/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml index 8b76dabbe7..260b93676e 100644 --- a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml @@ -40,6 +40,7 @@ conf: - fsspec~=2023.4.0 - numpy<2.0.0 - cryptography==42.0.5 + - opentelemetry-sdk~=1.32.1 name: momo-base-spark code: ../../src From 2fc212ce76aef2c0372fbb1b223864d09fa71db6 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 21:49:12 +0800 Subject: [PATCH 21/40] fix --- .../input_schema_adaptor/spec.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml index 260b93676e..1872442c88 100644 --- a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml @@ -35,12 +35,10 @@ conf: - azure-storage-file-datalake~=12.8.0 - azure-ai-ml~=1.26.3 - azureml-mlflow~=1.49.0 - - mltable~=1.3.0 - - azureml-fsspec~=1.0.0 + - mltable~=1.6.1 + - azureml-fsspec~=1.3.1 - fsspec~=2023.4.0 - numpy<2.0.0 - - cryptography==42.0.5 - - opentelemetry-sdk~=1.32.1 name: momo-base-spark code: ../../src From 563d8dd01be7e5d8f4043d80212f9cb1ad3df37e Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 8 May 2025 22:11:28 +0800 Subject: [PATCH 22/40] fix --- .../generation_safety_quality_signal_monitor/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml index a3d84b7158..a1a0ba0443 100644 --- a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml @@ -119,7 +119,7 @@ jobs: type: mltable resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.4" + runtime_version: "3.3" identity: type: aml_token compute_histogram: From f0ee51fc5880f0d0780285029b1dcfd2fae30a48 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 11:01:55 +0800 Subject: [PATCH 23/40] fix --- .../annotation_compute_histogram/spec.yaml | 17 +++-------------- .../spec.yaml | 6 +++--- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index dac995d708..6a86605971 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -102,7 +102,6 @@ conf: spark.jars.packages: com.microsoft.azure:synapseml_2.12:0.11.0-26-52919ce4-SNAPSHOT,org.apache.spark:spark-avro_2.12:3.3.1 spark.jars.repositories: https://mmlspark.azureedge.net/maven spark.jars.excludes: org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind - spark.hadoop.aml.enable_cache: true spark.yarn.user.classpath.first: true spark.sql.parquet.enableVectorizedReader: false spark.sql.legacy.replaceDatabricksSparkAvro.enabled: true @@ -114,19 +113,9 @@ conf: dependencies: - python=3.10 - pip: - - azure-cli-core~=2.66.0 - - azure-ai-evaluation~=1.0.1 - - azure-ai-ml~=1.26.3 - - openai~=1.56.2 - - json5==0.9.11 - - mltable~=1.6.1 - - keyrings.alt~=5.0.0 - - azureml-mlflow~=1.56.0 - - protobuf<5.29.0 - - mlflow~=2.17.2 - - azureml-fsspec~=1.3.1 - - fsspec~=2023.4.0 - - pyopenssl<23.0.0 + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 + - promptflow-devkit>=1.18.0 name: momo-gsq-spark code: ../../src entry: diff --git a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml index a1a0ba0443..a7f7182590 100644 --- a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml @@ -162,7 +162,7 @@ jobs: path: ${{parent.outputs.signal_scored_data}} resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.4" + runtime_version: "3.3" identity: type: aml_token compute_metrics: @@ -184,7 +184,7 @@ jobs: type: mltable resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.4" + runtime_version: "3.3" identity: type: aml_token output_signal_metrics: @@ -220,6 +220,6 @@ jobs: signal_name: ${{parent.inputs.signal_name}} resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.4" + runtime_version: "3.3" identity: type: aml_token From 8a3dcfbdfc8d28cd38600b532464c2649eb0f534 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 11:40:41 +0800 Subject: [PATCH 24/40] fix --- .../annotation_compute_histogram/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 6a86605971..9ffe4763b8 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -116,6 +116,7 @@ conf: - azure-ai-ml>=1.26.3 - openai>=1.56.2 - promptflow-devkit>=1.18.0 + - azure-ai-evaluation~=1.0.1 name: momo-gsq-spark code: ../../src entry: From 5f681e4706e2fc067dd431fbad0288fa1d397874 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 12:15:18 +0800 Subject: [PATCH 25/40] fix --- .../annotation_compute_histogram/spec.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 9ffe4763b8..605b2a86c0 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -117,6 +117,8 @@ conf: - openai>=1.56.2 - promptflow-devkit>=1.18.0 - azure-ai-evaluation~=1.0.1 + - azureml-dataprep 5.1.6 + name: momo-gsq-spark code: ../../src entry: From c7e58cbefdcc5557c4822036b134ae4cba6ab198 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 14:35:16 +0800 Subject: [PATCH 26/40] fix --- .../annotation_compute_histogram/spec.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 605b2a86c0..7ce987407b 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -117,7 +117,6 @@ conf: - openai>=1.56.2 - promptflow-devkit>=1.18.0 - azure-ai-evaluation~=1.0.1 - - azureml-dataprep 5.1.6 name: momo-gsq-spark code: ../../src From 39c1f564afaabd9fa121efc6c9936b59977d1e9a Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 14:45:28 +0800 Subject: [PATCH 27/40] fix --- .../annotation_compute_histogram/spec.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 7ce987407b..936ad52323 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -115,7 +115,6 @@ conf: - pip: - azure-ai-ml>=1.26.3 - openai>=1.56.2 - - promptflow-devkit>=1.18.0 - azure-ai-evaluation~=1.0.1 name: momo-gsq-spark From 111e38073e813de1567ee1367b3459de276d7616 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 16:40:08 +0800 Subject: [PATCH 28/40] fix --- .../annotation_compute_histogram/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 936ad52323..21f3b50bd5 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -116,6 +116,7 @@ conf: - azure-ai-ml>=1.26.3 - openai>=1.56.2 - azure-ai-evaluation~=1.0.1 + - azureml-dataprep==5.3.3 name: momo-gsq-spark code: ../../src From 3bb2d09e3b0d2f8cbd630ecbaeb6eff8858d335d Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 17:18:59 +0800 Subject: [PATCH 29/40] fix --- .../annotation_compute_histogram/spec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 21f3b50bd5..ef2818d78c 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -117,6 +117,7 @@ conf: - openai>=1.56.2 - azure-ai-evaluation~=1.0.1 - azureml-dataprep==5.3.3 + - mltable~=1.6.1 name: momo-gsq-spark code: ../../src From 2ae37c40a7cd69bf138f1eea4faac79b792ae320 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 17:52:43 +0800 Subject: [PATCH 30/40] fix --- .../annotation_compute_histogram/spec.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index ef2818d78c..1db8ce2d44 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -113,11 +113,11 @@ conf: dependencies: - python=3.10 - pip: - - azure-ai-ml>=1.26.3 - - openai>=1.56.2 - - azure-ai-evaluation~=1.0.1 - - azureml-dataprep==5.3.3 - - mltable~=1.6.1 + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 + - azure-ai-evaluation~=1.0.1 + - azureml-dataprep==5.3.3 + - mltable~=1.6.1 name: momo-gsq-spark code: ../../src From 5fe15bbb6fb16ce14813e6d316794ae20bb5d062 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 18:59:19 +0800 Subject: [PATCH 31/40] fix --- .../annotation_compute_histogram/spec.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 1db8ce2d44..ef2818d78c 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -113,11 +113,11 @@ conf: dependencies: - python=3.10 - pip: - - azure-ai-ml>=1.26.3 - - openai>=1.56.2 - - azure-ai-evaluation~=1.0.1 - - azureml-dataprep==5.3.3 - - mltable~=1.6.1 + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 + - azure-ai-evaluation~=1.0.1 + - azureml-dataprep==5.3.3 + - mltable~=1.6.1 name: momo-gsq-spark code: ../../src From 6d39fefe28320b014ea4590bdd11a74393c5b567 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 20:40:20 +0800 Subject: [PATCH 32/40] fix --- .../annotation_compute_histogram/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index ef2818d78c..83668a3f29 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -116,7 +116,7 @@ conf: - azure-ai-ml>=1.26.3 - openai>=1.56.2 - azure-ai-evaluation~=1.0.1 - - azureml-dataprep==5.3.3 + - azureml-dataprep==5.1.6 - mltable~=1.6.1 name: momo-gsq-spark From 1f55225a5194215dd78030d595db4351852f107b Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 22:57:24 +0800 Subject: [PATCH 33/40] fix --- .../annotation_compute_histogram/spec.yaml | 8 ++++++++ .../model_data_collector_preprocessor/spec.yaml | 1 + 2 files changed, 9 insertions(+) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 83668a3f29..9f3a3da35d 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -118,6 +118,14 @@ conf: - azure-ai-evaluation~=1.0.1 - azureml-dataprep==5.1.6 - mltable~=1.6.1 + - json5==0.9.11 + - keyrings.alt~=5.0.0 + - azureml-mlflow~=1.56.0 + - protobuf<5.29.0 + - mlflow~=2.17.2 + - azureml-fsspec~=1.3.1 + - fsspec~=2023.4.0 + - pyopenssl<23.0.0 name: momo-gsq-spark code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml index 5074803ac2..3b7874e7de 100644 --- a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml @@ -49,6 +49,7 @@ conf: - azureml-fsspec~=1.3.1 - fsspec~=2023.4.0 - numpy<2.0.0 + - cryptography==42.0.5 name: momo-base-spark args: >- --data_window_end ${{inputs.data_window_end}} From 5e4b99733c7cbd87e2a9684c20ef0f8c00ac4399 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Fri, 9 May 2025 23:40:29 +0800 Subject: [PATCH 34/40] fix --- .../model_monitor/model_data_collector_preprocessor/spec.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml index 3b7874e7de..5074803ac2 100644 --- a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml @@ -49,7 +49,6 @@ conf: - azureml-fsspec~=1.3.1 - fsspec~=2023.4.0 - numpy<2.0.0 - - cryptography==42.0.5 name: momo-base-spark args: >- --data_window_end ${{inputs.data_window_end}} From 08735e99d606a3fc7a6394922f5f5f5f07def21a Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Sat, 10 May 2025 01:28:37 +0800 Subject: [PATCH 35/40] fix --- .../annotation_compute_histogram/spec.yaml | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 9f3a3da35d..10ffd90ec4 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -113,20 +113,19 @@ conf: dependencies: - python=3.10 - pip: - - azure-ai-ml>=1.26.3 - - openai>=1.56.2 - - azure-ai-evaluation~=1.0.1 - - azureml-dataprep==5.1.6 - - mltable~=1.6.1 - - json5==0.9.11 - - keyrings.alt~=5.0.0 - - azureml-mlflow~=1.56.0 - - protobuf<5.29.0 - - mlflow~=2.17.2 - - azureml-fsspec~=1.3.1 - - fsspec~=2023.4.0 - - pyopenssl<23.0.0 - + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 + - azure-ai-evaluation~=1.0.1 + - azureml-dataprep==5.1.6 + - mltable~=1.6.1 + - json5==0.9.11 + - keyrings.alt~=5.0.0 + - azureml-mlflow~=1.56.0 + - protobuf<5.29.0 + - mlflow~=2.17.2 + - azureml-fsspec~=1.3.1 + - fsspec~=2023.4.0 + - pyopenssl<23.0.0 name: momo-gsq-spark code: ../../src entry: From 49af71dc91e03fd2631408881f299ea2c6365d8b Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Sat, 10 May 2025 01:28:58 +0800 Subject: [PATCH 36/40] fix --- .../annotation_compute_histogram/spec.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 10ffd90ec4..99caeec41c 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -113,19 +113,19 @@ conf: dependencies: - python=3.10 - pip: - - azure-ai-ml>=1.26.3 - - openai>=1.56.2 - - azure-ai-evaluation~=1.0.1 - - azureml-dataprep==5.1.6 - - mltable~=1.6.1 - - json5==0.9.11 - - keyrings.alt~=5.0.0 - - azureml-mlflow~=1.56.0 - - protobuf<5.29.0 - - mlflow~=2.17.2 - - azureml-fsspec~=1.3.1 - - fsspec~=2023.4.0 - - pyopenssl<23.0.0 + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 + - azure-ai-evaluation~=1.0.1 + - azureml-dataprep==5.1.6 + - mltable~=1.6.1 + - json5==0.9.11 + - keyrings.alt~=5.0.0 + - azureml-mlflow~=1.56.0 + - protobuf<5.29.0 + - mlflow~=2.17.2 + - azureml-fsspec~=1.3.1 + - fsspec~=2023.4.0 + - pyopenssl<23.0.0 name: momo-gsq-spark code: ../../src entry: From bcd661da754fc1db4bb7dbe2a7f66320de6437d4 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Sat, 10 May 2025 02:11:59 +0800 Subject: [PATCH 37/40] fix --- .../annotation_compute_histogram/spec.yaml | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 99caeec41c..86c9984528 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -113,19 +113,14 @@ conf: dependencies: - python=3.10 - pip: - - azure-ai-ml>=1.26.3 - - openai>=1.56.2 - - azure-ai-evaluation~=1.0.1 - - azureml-dataprep==5.1.6 - - mltable~=1.6.1 - - json5==0.9.11 - - keyrings.alt~=5.0.0 - - azureml-mlflow~=1.56.0 - - protobuf<5.29.0 - - mlflow~=2.17.2 - - azureml-fsspec~=1.3.1 - - fsspec~=2023.4.0 - - pyopenssl<23.0.0 + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 + - azure-ai-evaluation~=1.0.1 + - azureml-dataprep==5.1.6 + - mltable~=1.6.1 + - azureml-mlflow~=1.56.0 + - mlflow~=2.17.2 + - pyopenssl<23.0.0 name: momo-gsq-spark code: ../../src entry: From 6aa2646f11130784583856023ee79f5f571bec7c Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Sat, 10 May 2025 02:30:02 +0800 Subject: [PATCH 38/40] fix --- .../annotation_compute_histogram/spec.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index 86c9984528..22efb77bfa 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -118,9 +118,6 @@ conf: - azure-ai-evaluation~=1.0.1 - azureml-dataprep==5.1.6 - mltable~=1.6.1 - - azureml-mlflow~=1.56.0 - - mlflow~=2.17.2 - - pyopenssl<23.0.0 name: momo-gsq-spark code: ../../src entry: From 5313cadf0c219d808d378da8594211507ab6d509 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 13 May 2025 15:28:08 +0800 Subject: [PATCH 39/40] fix issue --- .../components/tests/e2e/conftest.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/assets/model_monitoring/components/tests/e2e/conftest.py b/assets/model_monitoring/components/tests/e2e/conftest.py index d47ac2ddcf..6734b8926e 100644 --- a/assets/model_monitoring/components/tests/e2e/conftest.py +++ b/assets/model_monitoring/components/tests/e2e/conftest.py @@ -13,9 +13,11 @@ from azure.ai.ml.constants import AssetTypes from azure.ai.ml.operations._run_history_constants import RunHistoryConstants, JobStatus from azure.ai.ml.entities import Job, Data +from azure.core.exceptions import ResourceNotFoundError from azure.identity import AzureCliCredential import pytest + from tests.e2e.utils.io_utils import ( write_to_yaml, load_from_yaml, @@ -56,15 +58,21 @@ def _watch_file(file: str, timeout_in_seconds): def _create_data_asset(ml_client: MLClient, name: str, path: str, type: AssetTypes): - my_data = Data( - path=path, - type=type, - name=name, - version='1' - ) - print(f"Creating a {type} data asset with name '{name}'") - ml_client.data.create_or_update(my_data) - + try: + ml_client.data.get(name=name, version='1') + print(f"Data asset '{name}' version '1' already exists.") + except ResourceNotFoundError: + my_data = Data( + path=path, + type=type, + name=name, + version='1' + ) + print(f"Creating a {type} data asset with name '{name}', version '1'") + ml_client.data.create_or_update(my_data) + except Exception as ex: + print(f"Unexpected error: {ex}") + raise @pytest.fixture(scope="session") def main_worker_lock(worker_id): From aff77f3a217af3c221a8b3e38d63e6908cd14e06 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 13 May 2025 17:28:31 +0800 Subject: [PATCH 40/40] fix --- assets/model_monitoring/components/tests/e2e/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/model_monitoring/components/tests/e2e/conftest.py b/assets/model_monitoring/components/tests/e2e/conftest.py index 6734b8926e..3dba18efa5 100644 --- a/assets/model_monitoring/components/tests/e2e/conftest.py +++ b/assets/model_monitoring/components/tests/e2e/conftest.py @@ -74,6 +74,7 @@ def _create_data_asset(ml_client: MLClient, name: str, path: str, type: AssetTyp print(f"Unexpected error: {ex}") raise + @pytest.fixture(scope="session") def main_worker_lock(worker_id): """Lock until the main worker releases its lock."""