From dfe9959bf6e2f72927f9ef7caad546ad03bdba01 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:00:33 +0800 Subject: [PATCH 01/14] Update Model Monitor version --- .../data_drift_compute_metrics/spec.yaml | 2 +- .../data_drift_signal_monitor/spec.yaml | 18 +++++++++--------- .../data_quality_compute_metrics/spec.yaml | 2 +- .../data_quality_metrics_joiner/spec.yaml | 2 +- .../data_quality_signal_monitor/spec.yaml | 18 +++++++++--------- .../data_quality_statistics/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 12 ++++++------ .../feature_importance_metrics/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 6 +++--- .../annotation_compute_histogram/spec.yaml | 2 +- .../annotation_compute_metrics/spec.yaml | 2 +- .../spec.yaml | 12 ++++++------ .../input_schema_adaptor/spec.yaml | 2 +- .../genai_mdc_preprocessor/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_monitor_compute_histogram/spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_monitor_create_manifest/spec.yaml | 2 +- .../model_monitor_data_joiner/spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_monitor_feature_selector/spec.yaml | 2 +- .../model_monitor_metric_outputter/spec.yaml | 2 +- .../model_monitor_output_metrics/spec.yaml | 2 +- .../action_analyzer_correlation_test/spec.yaml | 2 +- .../spec.yaml | 2 +- .../spec.yaml | 2 +- .../action_analyzer_output_actions/spec.yaml | 2 +- .../model_monitor_action_analyzer/spec.yaml | 10 +++++----- .../model_monitor_action_detector/spec.yaml | 2 +- .../spec.yaml | 2 +- .../model_performance_signal_monitor/spec.yaml | 8 ++++---- .../prediction_drift_signal_monitor/spec.yaml | 16 ++++++++-------- .../token_statistics_compute_metrics/spec.yaml | 2 +- 36 files changed, 78 insertions(+), 78 deletions(-) diff --git a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml index 21b982d43c..8045c5425b 100644 --- a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_drift_compute_metrics display_name: Data Drift - Compute Metrics description: Compute data drift metrics given a baseline and a deployment's model data input. -version: 0.3.33 +version: 0.3.34 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml index 3e09f8b0c6..c89728fea6 100644 --- a/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: data_drift_signal_monitor display_name: Data Drift - Signal Monitor description: Computes the data drift between a baseline and production data assets. -version: 0.3.54 +version: 0.3.55 is_deterministic: true inputs: @@ -63,7 +63,7 @@ outputs: jobs: compute_feature_importances: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -103,7 +103,7 @@ jobs: type: aml_token compute_drift_metrics: type: spark - component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.33 + component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.34 inputs: production_dataset: type: mltable @@ -130,7 +130,7 @@ jobs: type: aml_token compute_histogram_buckets: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.28 inputs: input_data_1: type: mltable @@ -150,7 +150,7 @@ jobs: type: aml_token compute_baseline_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -170,7 +170,7 @@ jobs: type: aml_token compute_target_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -190,7 +190,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -218,7 +218,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml index c01f1ce5b0..cc67916012 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_compute_metrics display_name: Data Quality - Compute Metrics description: Compute data quality metrics leveraged by the data quality monitor. -version: 0.3.32 +version: 0.3.33 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml index 52850a9483..9e38ca1118 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_metrics_joiner display_name: Data Quality - Metrics Joiner description: Join baseline and target data quality metrics into a single output. -version: 0.3.25 +version: 0.3.26 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml index 0f0bfce699..2e0295d8a1 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: data_quality_signal_monitor display_name: Data Quality - Signal Monitor description: Computes the data quality of a target dataset with reference to a baseline. -version: 0.3.52 +version: 0.3.53 is_deterministic: true inputs: @@ -63,7 +63,7 @@ outputs: jobs: compute_feature_importances: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -103,7 +103,7 @@ jobs: type: aml_token compute_baseline_data_statistics: type: spark - component: azureml://registries/azureml/components/data_quality_data_statistics/versions/0.3.27 + component: azureml://registries/azureml/components/data_quality_data_statistics/versions/0.3.28 inputs: baseline_data: type: mltable @@ -120,7 +120,7 @@ jobs: type: aml_token compute_baseline_data_quality: type: spark - component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.33 inputs: input_data: type: mltable @@ -145,7 +145,7 @@ jobs: type: aml_token compute_target_data_quality: type: spark - component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.33 inputs: input_data: type: mltable @@ -168,7 +168,7 @@ jobs: type: aml_token join_data_quality_metrics: type: spark - component: azureml://registries/azureml/components/data_quality_metrics_joiner/versions/0.3.25 + component: azureml://registries/azureml/components/data_quality_metrics_joiner/versions/0.3.26 inputs: baseline_metrics: type: mltable @@ -188,7 +188,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -210,7 +210,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml index e5e38dcbe2..5082770bb7 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_data_statistics display_name: Data Quality - Data Statistics description: Compute data statistics leveraged by the data quality monitor. -version: 0.3.27 +version: 0.3.28 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml index a4a5b7d4ab..4501cf2d82 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml @@ -2,7 +2,7 @@ $schema: http://azureml/sdk-2-0/SparkComponent.json type: spark name: feature_attribution_drift_compute_metrics -version: 0.3.28 +version: 0.3.29 display_name: Feature Attribution Drift - Compute Metrics is_deterministic: true description: Feature attribution drift using model monitoring. diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml index 8e47e34117..e3ad247fdc 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: feature_attribution_drift_signal_monitor display_name: Feature Attribution Drift - Signal Monitor description: Computes the feature attribution between a baseline and production data assets. -version: 0.3.44 +version: 0.3.45 is_deterministic: true inputs: @@ -44,7 +44,7 @@ outputs: jobs: compute_baseline_explanations: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -63,7 +63,7 @@ jobs: type: aml_token compute_production_explanations: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token compute_feature_attribution: type: spark - component: azureml://registries/azureml/components/feature_attribution_drift_compute_metrics/versions/0.3.28 + component: azureml://registries/azureml/components/feature_attribution_drift_compute_metrics/versions/0.3.29 inputs: production_data: type: mltable @@ -101,7 +101,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -122,7 +122,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml index 661f7990c4..f251573d39 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml @@ -2,7 +2,7 @@ $schema: http://azureml/sdk-2-0/SparkComponent.json type: spark name: feature_importance_metrics -version: 0.3.31 +version: 0.3.32 display_name: Feature importance is_deterministic: true description: Feature importance for model monitoring. diff --git a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml index a3b5812c2f..7716d3d56e 100644 --- a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: genai_token_statistics_compute_metrics display_name: GenAI Token Statistics - Compute Metrics description: Compute token statistics metrics. -version: 0.0.21 +version: 0.0.22 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml index 9a88b12c12..fa775d2a86 100644 --- a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: genai_token_statistics_signal_monitor display_name: GenAI Token Statistics - Signal Monitor description: Computes the token and cost metrics over LLM outputs. -version: 0.0.21 +version: 0.0.22 is_deterministic: true inputs: monitor_name: @@ -30,7 +30,7 @@ outputs: jobs: compute_metrics: type: spark - component: azureml://registries/azureml/components/genai_token_statistics_compute_metrics/versions/0.0.21 + component: azureml://registries/azureml/components/genai_token_statistics_compute_metrics/versions/0.0.22 inputs: production_dataset: type: mltable @@ -49,7 +49,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index d7d7442008..b5570b7957 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_annotation_compute_histogram display_name: Annotation - Compute Histogram description: Compute annotation histogram given a deployment's model data input. -version: 0.4.36 +version: 0.4.37 is_deterministic: false inputs: production_dataset: diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml index 5f52d05793..a809425699 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_annotation_compute_metrics display_name: Annotation - Compute Metrics description: Compute annotation metrics given a deployment's model data input. -version: 0.4.29 +version: 0.4.30 is_deterministic: True inputs: annotation_histogram: diff --git a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml index cf9c1afc9d..de338175a4 100644 --- a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: generation_safety_quality_signal_monitor display_name: Generation Safety & Quality - Signal Monitor description: Computes the content generation safety metrics over LLM outputs. -version: 0.5.27 +version: 0.5.28 is_deterministic: true inputs: monitor_name: @@ -109,7 +109,7 @@ outputs: jobs: input_schema_adaptor: type: spark - component: azureml://registries/azureml/components/gsq_input_schema_adaptor/versions/0.0.24 + component: azureml://registries/azureml/components/gsq_input_schema_adaptor/versions/0.0.25 inputs: production_dataset: type: mltable @@ -124,7 +124,7 @@ jobs: type: aml_token compute_histogram: type: spark - component: azureml://registries/azureml/components/gsq_annotation_compute_histogram/versions/0.4.36 + component: azureml://registries/azureml/components/gsq_annotation_compute_histogram/versions/0.4.37 inputs: production_dataset: type: mltable @@ -167,7 +167,7 @@ jobs: type: aml_token compute_metrics: type: spark - component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.29 + component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.30 inputs: annotation_histogram: type: mltable @@ -189,7 +189,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable @@ -211,7 +211,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml index 60cbab7f0e..cc6af8ff08 100644 --- a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_input_schema_adaptor display_name: Input Schema Adaptor description: Adapt data to fit into GSQ component. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: production_dataset: diff --git a/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml index 07a80d385a..c1f8dfa494 100644 --- a/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: genai_mdc_preprocessor display_name: GenAI MDC - Preprocessor description: Filters the raw span log based on the window provided, and aggregates it to trace level. -version: 0.0.28 +version: 0.0.29 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml index 95c86b4da3..5074803ac2 100644 --- a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_data_collector_preprocessor display_name: Model Data Collector - Preprocessor description: Filters the data based on the window provided. -version: 0.4.29 +version: 0.4.30 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml index 96358f208a..aa7006fa0e 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_azmon_metric_publisher display_name: Model Monitor - Azure Monitor Metric Publisher description: Azure Monitor Publisher for the computed model monitor metrics. -version: 0.3.31 +version: 0.3.32 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml index bea674b797..40946e57a9 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_compute_histogram display_name: Model Monitor - Compute Histogram description: Compute a histogram given an input data and associated histogram buckets. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml index f1d18eb705..926d631c58 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_compute_histogram_buckets display_name: Model Monitor - Compute Histogram Buckets description: Compute histogram buckets given up to two datasets. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml index ba25c5f5cb..e5a41373c7 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_create_manifest display_name: Model Monitor - Create Manifest description: Creates the model monitor metric manifest. -version: 0.3.24 +version: 0.3.25 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml index 0c07c9735b..9b322230e2 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_data_joiner display_name: Model Monitor - Data Joiner description: Joins two data assets on the given columns for model monitor. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml index 713342d63e..8d124db6f2 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_evaluate_metrics_threshold display_name: Model Monitor - Evaluate Metrics Threshold description: Evaluate signal metrics against the threshold provided in the monitoring signal. -version: 0.3.32 +version: 0.3.33 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml index e24e594285..30c18ac735 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_feature_selector display_name: Model Monitor - Feature Selector description: Selects features to compute signal metrics on. -version: 0.3.28 +version: 0.3.29 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml index 722dcfe219..83f172e9cc 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_metric_outputter display_name: Model Monitor - Metric Outputter description: Output the computed model monitor metrics. -version: 0.3.35 +version: 0.3.36 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml index c1e828fbbe..1c1507eb1f 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_output_metrics display_name: Model Monitor - Output Metrics description: Output the computed model monitor metrics to the default datastore. -version: 0.3.32 +version: 0.3.33 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml index 8c8b8f84de..0c9dcae963 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_correlation_test display_name: Action Analyzer - Correlation Test description: Perform correlation test on different groups to generate actions. -version: 0.0.21 +version: 0.0.22 is_deterministic: True inputs: data_with_action_metric_score: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml index 0b9b97cd39..67b5bdfada 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_identify_problem_traffic display_name: Action Analyzer - Identify Problem Traffic description: Separate bad queries into different groups. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: signal_output: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml index 658714a7fc..f054b6b3ae 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_metrics_calculation display_name: Action Analyzer - Metrics Calculation description: Calculate futher metrics for generating actions. -version: 0.0.21 +version: 0.0.22 is_deterministic: True inputs: data_with_groups: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml index 7bd80a94ab..6c15cddc1e 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_output_actions display_name: Action Analyzer - Output Actions description: Merge and output actions. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: action_data: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml index 42d2d8bb59..42e8f567eb 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: model_monitor_action_analyzer display_name: Model Monitor - Action Analyzer description: Generate and output actions to the default datastore. -version: 0.0.25 +version: 0.0.26 is_deterministic: true inputs: signal_output: @@ -52,7 +52,7 @@ outputs: jobs: identify_problem_traffic: type: spark - component: azureml://registries/azureml/components/action_analyzer_identify_problem_traffic/versions/0.0.24 + component: azureml://registries/azureml/components/action_analyzer_identify_problem_traffic/versions/0.0.25 inputs: signal_output: type: uri_folder @@ -74,7 +74,7 @@ jobs: type: aml_token metrics_calculation: type: spark - component: azureml://registries/azureml/components/action_analyzer_metrics_calculation/versions/0.0.21 + component: azureml://registries/azureml/components/action_analyzer_metrics_calculation/versions/0.0.22 inputs: data_with_groups: type: mltable @@ -91,7 +91,7 @@ jobs: type: aml_token correlation_test: type: spark - component: azureml://registries/azureml/components/action_analyzer_correlation_test/versions/0.0.21 + component: azureml://registries/azureml/components/action_analyzer_correlation_test/versions/0.0.22 inputs: data_with_action_metric_score: type: mltable @@ -106,7 +106,7 @@ jobs: type: aml_token output_actions: type: spark - component: azureml://registries/azureml/components/action_analyzer_output_actions/versions/0.0.24 + component: azureml://registries/azureml/components/action_analyzer_output_actions/versions/0.0.25 inputs: action_data: type: mltable diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml index 467321060c..ec21779dde 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_action_detector display_name: Model Monitor - Action Detector description: Generate and output actions -version: 0.0.17 +version: 0.0.18 is_deterministic: true inputs: signal_output: diff --git a/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml b/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml index 7650b59b1c..8bfe429cff 100644 --- a/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_performance_compute_metrics display_name: Model Performance - Compute Metrics description: Compute model performance metrics leveraged by the model performance monitor. -version: 0.0.26 +version: 0.0.27 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml b/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml index 766d3608f8..da8e745b64 100644 --- a/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: model_performance_signal_monitor display_name: Model Performance - Signal Monitor description: Computes the model performance -version: 0.0.28 +version: 0.0.29 is_deterministic: true inputs: task: @@ -57,7 +57,7 @@ outputs: jobs: compute_metrics: type: spark - component: azureml://registries/azureml/components/model_performance_compute_metrics/versions/0.0.26 + component: azureml://registries/azureml/components/model_performance_compute_metrics/versions/0.0.27 inputs: task: ${{parent.inputs.task}} baseline_data_target_column: ${{parent.inputs.baseline_data_target_column}} @@ -78,7 +78,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable @@ -99,7 +99,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml index b2cdafdf3e..094db6d9f4 100644 --- a/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: prediction_drift_signal_monitor display_name: Prediction Drift - Signal Monitor description: Computes the prediction drift between a baseline and a target data assets. -version: 0.4.30 +version: 0.4.31 is_deterministic: true inputs: @@ -57,7 +57,7 @@ outputs: jobs: feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -77,7 +77,7 @@ jobs: type: aml_token compute_drift_metrics: type: spark - component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.33 + component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.34 inputs: production_dataset: type: mltable @@ -104,7 +104,7 @@ jobs: type: aml_token compute_histogram_buckets: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.28 inputs: input_data_1: type: mltable @@ -124,7 +124,7 @@ jobs: type: aml_token compute_baseline_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -144,7 +144,7 @@ jobs: type: aml_token compute_target_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -164,7 +164,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -191,7 +191,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml b/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml index 07c65e989c..338c5322cd 100644 --- a/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: token_statistics_compute_metrics display_name: Token Statistics - Compute Metrics description: Compute token statistics metrics. -version: 0.0.23 +version: 0.0.24 is_deterministic: true inputs: From 29d26f37d5969769e3cfb58c3b4e6f981f9a26d9 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 00:23:52 +0800 Subject: [PATCH 02/14] debug --- .../e2e/test_generation_safety_quality_e2e.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index afd190fecc..5536f50de1 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -96,6 +96,19 @@ def test_generation_safety_quality_successful( } ) + if pipeline_job.status != "Completed": + # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 + print("Pipeline job failed!") + print(f"Status: {pipeline_job.status}") + # 打印详细错误信息（属性名可能不同，具体可以 dir(pipeline_job) 看下） + if hasattr(pipeline_job, "error"): + print(f"Error: {pipeline_job.error}") + if hasattr(pipeline_job, "details"): + print(f"Details: {pipeline_job.details}") + # 你也可以打印日志链接等 + if hasattr(pipeline_job, "studio_url"): + print(f"Check logs at: {pipeline_job.studio_url}") + assert pipeline_job.status == "Completed" def test_generation_safety_quality_genai_successful( @@ -115,4 +128,17 @@ def test_generation_safety_quality_genai_successful( } ) + if pipeline_job.status != "Completed": + # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 + print("Pipeline job failed!") + print(f"Status: {pipeline_job.status}") + # 打印详细错误信息（属性名可能不同，具体可以 dir(pipeline_job) 看下） + if hasattr(pipeline_job, "error"): + print(f"Error: {pipeline_job.error}") + if hasattr(pipeline_job, "details"): + print(f"Details: {pipeline_job.details}") + # 你也可以打印日志链接等 + if hasattr(pipeline_job, "studio_url"): + print(f"Check logs at: {pipeline_job.studio_url}") + assert pipeline_job.status == "Completed" From 9f5b4dbdc1912d665d40eddfb9df0c006e82a450 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 15:55:14 +0800 Subject: [PATCH 03/14] fix --- .../tests/e2e/test_generation_safety_quality_e2e.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index 5536f50de1..c7f4db1543 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -95,6 +95,12 @@ def test_generation_safety_quality_successful( "completion_column_name": "answer", } ) + print(f"\n==== Azure ML Context ====") + print(f"Workspace : {ml_client.workspace_name}") + print(f"Resource Group : {ml_client.resource_group_name}") + print(f"Subscription : {ml_client.subscription_id}") + print(f"=========================\n") + if pipeline_job.status != "Completed": # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 @@ -127,6 +133,12 @@ def test_generation_safety_quality_genai_successful( "context_column_name": "context", } ) + print(f"\n==== Azure ML Context ====") + print(f"Workspace : {ml_client.workspace_name}") + print(f"Resource Group : {ml_client.resource_group_name}") + print(f"Subscription : {ml_client.subscription_id}") + print(f"=========================\n") + if pipeline_job.status != "Completed": # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 From 4aa119cc3ebfa12ada3a54d2f3dc5ad356d498d9 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 21:09:10 +0800 Subject: [PATCH 04/14] fix --- .github/workflows/model-monitoring-ci.yml | 14 ++++++++++++++ .../e2e/test_generation_safety_quality_e2e.py | 9 ++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index 75d23c4e61..ccabda6b8c 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -56,6 +56,20 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} + - name: Grant Storage Blob Data Contributor to AML Workspace MSI + run: | + AML_WORKSPACE_NAME=${{ env.workspace }} + RESOURCE_GROUP=${{ env.resource_group }} + STORAGE_ACCOUNT_NAME=azuremlastorage06934ba65 + + # 获取AML Workspace的MSI对象ID + MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -w "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) + + # 分配权限 + az role assignment create \ + --assignee-object-id "$MSI_PRINCIPAL_ID" \ + --role "Storage Blob Data Contributor" \ + --scope "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$STORAGE_ACCOUNT_NAME" - name: Publish Component Specs run: python -m pytest ${{ env.testsRootPath }}/e2e/utils/test_publish_components_for_ci.py::TestPublishComponentsForCI::test_publish_components env: diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index c7f4db1543..ca44e4680a 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -96,12 +96,11 @@ def test_generation_safety_quality_successful( } ) print(f"\n==== Azure ML Context ====") - print(f"Workspace : {ml_client.workspace_name}") + print(f"Workspace : {ml_client.workspace_name}") print(f"Resource Group : {ml_client.resource_group_name}") - print(f"Subscription : {ml_client.subscription_id}") + print(f"Subscription : {ml_client.subscription_id}") print(f"=========================\n") - if pipeline_job.status != "Completed": # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 print("Pipeline job failed!") @@ -134,9 +133,9 @@ def test_generation_safety_quality_genai_successful( } ) print(f"\n==== Azure ML Context ====") - print(f"Workspace : {ml_client.workspace_name}") + print(f"Workspace : {ml_client.workspace_name}") print(f"Resource Group : {ml_client.resource_group_name}") - print(f"Subscription : {ml_client.subscription_id}") + print(f"Subscription : {ml_client.subscription_id}") print(f"=========================\n") From 8b549dc91f522219adad8c6693bba5958ccaaa9e Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 21:44:33 +0800 Subject: [PATCH 05/14] fix --- .github/workflows/model-monitoring-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index ccabda6b8c..e264d89140 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -56,20 +56,23 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} + - name: Grant Storage Blob Data Contributor to AML Workspace MSI run: | AML_WORKSPACE_NAME=${{ env.workspace }} RESOURCE_GROUP=${{ env.resource_group }} STORAGE_ACCOUNT_NAME=azuremlastorage06934ba65 + SUBSCRIPTION_ID=${{ env.subscription_id }} # 获取AML Workspace的MSI对象ID - MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -w "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) + MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) # 分配权限 az role assignment create \ --assignee-object-id "$MSI_PRINCIPAL_ID" \ --role "Storage Blob Data Contributor" \ - --scope "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$STORAGE_ACCOUNT_NAME" + --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$STORAGE_ACCOUNT_NAME" + - name: Publish Component Specs run: python -m pytest ${{ env.testsRootPath }}/e2e/utils/test_publish_components_for_ci.py::TestPublishComponentsForCI::test_publish_components env: From b5ec3dc564868d576da3f97e88e809103d4b8527 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 21:49:18 +0800 Subject: [PATCH 06/14] fix --- .github/workflows/model-monitoring-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index e264d89140..e6ef5fa69c 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -56,7 +56,7 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} - + - name: Grant Storage Blob Data Contributor to AML Workspace MSI run: | AML_WORKSPACE_NAME=${{ env.workspace }} @@ -69,7 +69,7 @@ jobs: # 分配权限 az role assignment create \ - --assignee-object-id "$MSI_PRINCIPAL_ID" \ + --assignee "$MSI_PRINCIPAL_ID" \ --role "Storage Blob Data Contributor" \ --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$STORAGE_ACCOUNT_NAME" From 43cb9b2effe059bbe1855d9ea9d2ff293a4baf40 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 22:21:52 +0800 Subject: [PATCH 07/14] fix --- .github/workflows/model-monitoring-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index e6ef5fa69c..0b3cfcac9b 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -67,6 +67,11 @@ jobs: # 获取AML Workspace的MSI对象ID MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) + az ml workspace update \ + --resource-group my-resource-group \ + --name my-ml-workspace \ + --set identity.type=SystemAssigned + # 分配权限 az role assignment create \ --assignee "$MSI_PRINCIPAL_ID" \ From 3334fadd8e5d0aef5a828eb06391be8173a603df Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 22:25:53 +0800 Subject: [PATCH 08/14] fix --- .github/workflows/model-monitoring-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index 0b3cfcac9b..415bfc4a2a 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -68,8 +68,8 @@ jobs: MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) az ml workspace update \ - --resource-group my-resource-group \ - --name my-ml-workspace \ + --resource-group $RESOURCE_GROUP \ + --name $AML_WORKSPACE_NAME \ --set identity.type=SystemAssigned # 分配权限 From 70e3eb246fb633e03fd5044a4cd9e3c5345c7fbe Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 22:34:28 +0800 Subject: [PATCH 09/14] fix --- .github/workflows/model-monitoring-ci.yml | 2 - .../e2e/test_generation_safety_quality_e2e.py | 37 ------------------- 2 files changed, 39 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index 415bfc4a2a..7184872a9d 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -64,7 +64,6 @@ jobs: STORAGE_ACCOUNT_NAME=azuremlastorage06934ba65 SUBSCRIPTION_ID=${{ env.subscription_id }} - # 获取AML Workspace的MSI对象ID MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) az ml workspace update \ @@ -72,7 +71,6 @@ jobs: --name $AML_WORKSPACE_NAME \ --set identity.type=SystemAssigned - # 分配权限 az role assignment create \ --assignee "$MSI_PRINCIPAL_ID" \ --role "Storage Blob Data Contributor" \ diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index ca44e4680a..afd190fecc 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -95,24 +95,6 @@ def test_generation_safety_quality_successful( "completion_column_name": "answer", } ) - print(f"\n==== Azure ML Context ====") - print(f"Workspace : {ml_client.workspace_name}") - print(f"Resource Group : {ml_client.resource_group_name}") - print(f"Subscription : {ml_client.subscription_id}") - print(f"=========================\n") - - if pipeline_job.status != "Completed": - # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 - print("Pipeline job failed!") - print(f"Status: {pipeline_job.status}") - # 打印详细错误信息（属性名可能不同，具体可以 dir(pipeline_job) 看下） - if hasattr(pipeline_job, "error"): - print(f"Error: {pipeline_job.error}") - if hasattr(pipeline_job, "details"): - print(f"Details: {pipeline_job.details}") - # 你也可以打印日志链接等 - if hasattr(pipeline_job, "studio_url"): - print(f"Check logs at: {pipeline_job.studio_url}") assert pipeline_job.status == "Completed" @@ -132,24 +114,5 @@ def test_generation_safety_quality_genai_successful( "context_column_name": "context", } ) - print(f"\n==== Azure ML Context ====") - print(f"Workspace : {ml_client.workspace_name}") - print(f"Resource Group : {ml_client.resource_group_name}") - print(f"Subscription : {ml_client.subscription_id}") - print(f"=========================\n") - - - if pipeline_job.status != "Completed": - # 尝试打印详细信息，属性名可能需要根据你的 pipeline_job 类型调整 - print("Pipeline job failed!") - print(f"Status: {pipeline_job.status}") - # 打印详细错误信息（属性名可能不同，具体可以 dir(pipeline_job) 看下） - if hasattr(pipeline_job, "error"): - print(f"Error: {pipeline_job.error}") - if hasattr(pipeline_job, "details"): - print(f"Details: {pipeline_job.details}") - # 你也可以打印日志链接等 - if hasattr(pipeline_job, "studio_url"): - print(f"Check logs at: {pipeline_job.studio_url}") assert pipeline_job.status == "Completed" From aeeb4f9e3035967cbd1685d99822783408e28e5a Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 22:41:15 +0800 Subject: [PATCH 10/14] fix --- .github/workflows/model-monitoring-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index 7184872a9d..a6b3d7b747 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -64,13 +64,13 @@ jobs: STORAGE_ACCOUNT_NAME=azuremlastorage06934ba65 SUBSCRIPTION_ID=${{ env.subscription_id }} - MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) - az ml workspace update \ --resource-group $RESOURCE_GROUP \ --name $AML_WORKSPACE_NAME \ --set identity.type=SystemAssigned + MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) + az role assignment create \ --assignee "$MSI_PRINCIPAL_ID" \ --role "Storage Blob Data Contributor" \ From 195f29f9fa06af10e69436f1337e4c13d85ed6f1 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Mon, 5 May 2025 22:51:26 +0800 Subject: [PATCH 11/14] fix --- .github/workflows/model-monitoring-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index a6b3d7b747..522dbc06b1 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -69,7 +69,7 @@ jobs: --name $AML_WORKSPACE_NAME \ --set identity.type=SystemAssigned - MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principalId" -o tsv) + MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principal_id" -o tsv) az role assignment create \ --assignee "$MSI_PRINCIPAL_ID" \ From 7bd559f4f89967535d177740b8b8f2cd7e7b8cd8 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 18:56:22 +0800 Subject: [PATCH 12/14] fix --- .github/workflows/model-monitoring-ci.yml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index 522dbc06b1..43e9941e66 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -57,25 +57,9 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} - - name: Grant Storage Blob Data Contributor to AML Workspace MSI - run: | - AML_WORKSPACE_NAME=${{ env.workspace }} - RESOURCE_GROUP=${{ env.resource_group }} - STORAGE_ACCOUNT_NAME=azuremlastorage06934ba65 - SUBSCRIPTION_ID=${{ env.subscription_id }} - - az ml workspace update \ - --resource-group $RESOURCE_GROUP \ - --name $AML_WORKSPACE_NAME \ - --set identity.type=SystemAssigned - - MSI_PRINCIPAL_ID=$(az ml workspace show -g "$RESOURCE_GROUP" -n "$AML_WORKSPACE_NAME" --query "identity.principal_id" -o tsv) - - az role assignment create \ - --assignee "$MSI_PRINCIPAL_ID" \ - --role "Storage Blob Data Contributor" \ - --scope "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.Storage/storageAccounts/$STORAGE_ACCOUNT_NAME" - + - name: Print subscription id + run: echo "Subscription id is ${{ secrets.AZURE_SUBSCRIPTION_ID }}" + - name: Publish Component Specs run: python -m pytest ${{ env.testsRootPath }}/e2e/utils/test_publish_components_for_ci.py::TestPublishComponentsForCI::test_publish_components env: From 129400efef6d1b094ea3ac5952913846d270fb57 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 19:07:04 +0800 Subject: [PATCH 13/14] fix --- .github/workflows/model-monitoring-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index 43e9941e66..e67d83d542 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -58,7 +58,9 @@ jobs: scripts-setup-dir: ${{ env.scripts_setup_dir }} - name: Print subscription id - run: echo "Subscription id is ${{ secrets.AZURE_SUBSCRIPTION_ID }}" + run: echo "Subscription id length is ${AZURE_SUBSCRIPTION_ID}" + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Publish Component Specs run: python -m pytest ${{ env.testsRootPath }}/e2e/utils/test_publish_components_for_ci.py::TestPublishComponentsForCI::test_publish_components From 694f9f23b7f983129f55412d15ab9771a8d50455 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Tue, 6 May 2025 19:26:43 +0800 Subject: [PATCH 14/14] fix --- .github/workflows/model-monitoring-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/model-monitoring-ci.yml b/.github/workflows/model-monitoring-ci.yml index e67d83d542..137f80274c 100644 --- a/.github/workflows/model-monitoring-ci.yml +++ b/.github/workflows/model-monitoring-ci.yml @@ -57,10 +57,9 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} scripts-setup-dir: ${{ env.scripts_setup_dir }} - - name: Print subscription id - run: echo "Subscription id length is ${AZURE_SUBSCRIPTION_ID}" - env: - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Azure CLI show current subscription + run: az account show --query id -o tsv + - name: Publish Component Specs run: python -m pytest ${{ env.testsRootPath }}/e2e/utils/test_publish_components_for_ci.py::TestPublishComponentsForCI::test_publish_components