diff --git a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml index 21b982d43c..8045c5425b 100644 --- a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_drift_compute_metrics display_name: Data Drift - Compute Metrics description: Compute data drift metrics given a baseline and a deployment's model data input. -version: 0.3.33 +version: 0.3.34 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml index 3e09f8b0c6..c89728fea6 100644 --- a/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/data_drift/data_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: data_drift_signal_monitor display_name: Data Drift - Signal Monitor description: Computes the data drift between a baseline and production data assets. -version: 0.3.54 +version: 0.3.55 is_deterministic: true inputs: @@ -63,7 +63,7 @@ outputs: jobs: compute_feature_importances: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -103,7 +103,7 @@ jobs: type: aml_token compute_drift_metrics: type: spark - component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.33 + component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.34 inputs: production_dataset: type: mltable @@ -130,7 +130,7 @@ jobs: type: aml_token compute_histogram_buckets: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.28 inputs: input_data_1: type: mltable @@ -150,7 +150,7 @@ jobs: type: aml_token compute_baseline_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -170,7 +170,7 @@ jobs: type: aml_token compute_target_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -190,7 +190,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -218,7 +218,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml index c01f1ce5b0..cc67916012 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_compute_metrics display_name: Data Quality - Compute Metrics description: Compute data quality metrics leveraged by the data quality monitor. -version: 0.3.32 +version: 0.3.33 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml index 52850a9483..9e38ca1118 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_metrics_joiner display_name: Data Quality - Metrics Joiner description: Join baseline and target data quality metrics into a single output. -version: 0.3.25 +version: 0.3.26 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml index 0f0bfce699..2e0295d8a1 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: data_quality_signal_monitor display_name: Data Quality - Signal Monitor description: Computes the data quality of a target dataset with reference to a baseline. -version: 0.3.52 +version: 0.3.53 is_deterministic: true inputs: @@ -63,7 +63,7 @@ outputs: jobs: compute_feature_importances: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -103,7 +103,7 @@ jobs: type: aml_token compute_baseline_data_statistics: type: spark - component: azureml://registries/azureml/components/data_quality_data_statistics/versions/0.3.27 + component: azureml://registries/azureml/components/data_quality_data_statistics/versions/0.3.28 inputs: baseline_data: type: mltable @@ -120,7 +120,7 @@ jobs: type: aml_token compute_baseline_data_quality: type: spark - component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.33 inputs: input_data: type: mltable @@ -145,7 +145,7 @@ jobs: type: aml_token compute_target_data_quality: type: spark - component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/data_quality_compute_metrics/versions/0.3.33 inputs: input_data: type: mltable @@ -168,7 +168,7 @@ jobs: type: aml_token join_data_quality_metrics: type: spark - component: azureml://registries/azureml/components/data_quality_metrics_joiner/versions/0.3.25 + component: azureml://registries/azureml/components/data_quality_metrics_joiner/versions/0.3.26 inputs: baseline_metrics: type: mltable @@ -188,7 +188,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -210,7 +210,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml index e5e38dcbe2..5082770bb7 100644 --- a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml +++ b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: data_quality_data_statistics display_name: Data Quality - Data Statistics description: Compute data statistics leveraged by the data quality monitor. -version: 0.3.27 +version: 0.3.28 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml index a4a5b7d4ab..4501cf2d82 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml @@ -2,7 +2,7 @@ $schema: http://azureml/sdk-2-0/SparkComponent.json type: spark name: feature_attribution_drift_compute_metrics -version: 0.3.28 +version: 0.3.29 display_name: Feature Attribution Drift - Compute Metrics is_deterministic: true description: Feature attribution drift using model monitoring. diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml index 8e47e34117..e3ad247fdc 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_attribution_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: feature_attribution_drift_signal_monitor display_name: Feature Attribution Drift - Signal Monitor description: Computes the feature attribution between a baseline and production data assets. -version: 0.3.44 +version: 0.3.45 is_deterministic: true inputs: @@ -44,7 +44,7 @@ outputs: jobs: compute_baseline_explanations: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -63,7 +63,7 @@ jobs: type: aml_token compute_production_explanations: type: spark - component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.31 + component: azureml://registries/azureml/components/feature_importance_metrics/versions/0.3.32 inputs: baseline_data: type: mltable @@ -82,7 +82,7 @@ jobs: type: aml_token compute_feature_attribution: type: spark - component: azureml://registries/azureml/components/feature_attribution_drift_compute_metrics/versions/0.3.28 + component: azureml://registries/azureml/components/feature_attribution_drift_compute_metrics/versions/0.3.29 inputs: production_data: type: mltable @@ -101,7 +101,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -122,7 +122,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml b/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml index 661f7990c4..f251573d39 100644 --- a/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml +++ b/assets/model_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml @@ -2,7 +2,7 @@ $schema: http://azureml/sdk-2-0/SparkComponent.json type: spark name: feature_importance_metrics -version: 0.3.31 +version: 0.3.32 display_name: Feature importance is_deterministic: true description: Feature importance for model monitoring. diff --git a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml index a3b5812c2f..7716d3d56e 100644 --- a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: genai_token_statistics_compute_metrics display_name: GenAI Token Statistics - Compute Metrics description: Compute token statistics metrics. -version: 0.0.21 +version: 0.0.22 is_deterministic: true inputs: diff --git a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml index 9a88b12c12..fa775d2a86 100644 --- a/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/genai_token_statistics/genai_token_statistics_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: genai_token_statistics_signal_monitor display_name: GenAI Token Statistics - Signal Monitor description: Computes the token and cost metrics over LLM outputs. -version: 0.0.21 +version: 0.0.22 is_deterministic: true inputs: monitor_name: @@ -30,7 +30,7 @@ outputs: jobs: compute_metrics: type: spark - component: azureml://registries/azureml/components/genai_token_statistics_compute_metrics/versions/0.0.21 + component: azureml://registries/azureml/components/genai_token_statistics_compute_metrics/versions/0.0.22 inputs: production_dataset: type: mltable @@ -49,7 +49,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml index d7d7442008..22efb77bfa 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_annotation_compute_histogram display_name: Annotation - Compute Histogram description: Compute annotation histogram given a deployment's model data input. -version: 0.4.36 +version: 0.4.37 is_deterministic: false inputs: production_dataset: @@ -113,19 +113,11 @@ conf: dependencies: - python=3.10 - pip: - - azure-cli-core~=2.66.0 + - azure-ai-ml>=1.26.3 + - openai>=1.56.2 - azure-ai-evaluation~=1.0.1 - - azure-ai-ml~=1.26.3 - - openai~=1.56.2 - - json5==0.9.11 + - azureml-dataprep==5.1.6 - mltable~=1.6.1 - - keyrings.alt~=5.0.0 - - azureml-mlflow~=1.56.0 - - protobuf<5.29.0 - - mlflow~=2.17.2 - - azureml-fsspec~=1.3.1 - - fsspec~=2023.4.0 - - pyopenssl<23.0.0 name: momo-gsq-spark code: ../../src entry: diff --git a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml index 5f52d05793..a809425699 100644 --- a/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_annotation_compute_metrics display_name: Annotation - Compute Metrics description: Compute annotation metrics given a deployment's model data input. -version: 0.4.29 +version: 0.4.30 is_deterministic: True inputs: annotation_histogram: diff --git a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml index cf9c1afc9d..a7f7182590 100644 --- a/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/generation_safety_quality_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: generation_safety_quality_signal_monitor display_name: Generation Safety & Quality - Signal Monitor description: Computes the content generation safety metrics over LLM outputs. -version: 0.5.27 +version: 0.5.28 is_deterministic: true inputs: monitor_name: @@ -109,7 +109,7 @@ outputs: jobs: input_schema_adaptor: type: spark - component: azureml://registries/azureml/components/gsq_input_schema_adaptor/versions/0.0.24 + component: azureml://registries/azureml/components/gsq_input_schema_adaptor/versions/0.0.25 inputs: production_dataset: type: mltable @@ -124,7 +124,7 @@ jobs: type: aml_token compute_histogram: type: spark - component: azureml://registries/azureml/components/gsq_annotation_compute_histogram/versions/0.4.36 + component: azureml://registries/azureml/components/gsq_annotation_compute_histogram/versions/0.4.37 inputs: production_dataset: type: mltable @@ -167,7 +167,7 @@ jobs: type: aml_token compute_metrics: type: spark - component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.29 + component: azureml://registries/azureml/components/gsq_annotation_compute_metrics/versions/0.4.30 inputs: annotation_histogram: type: mltable @@ -189,7 +189,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable @@ -206,12 +206,12 @@ jobs: mode: direct resources: instance_type: ${{parent.inputs.instance_type}} - runtime_version: "3.3" + runtime_version: "3.4" identity: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml index 60cbab7f0e..1872442c88 100644 --- a/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml +++ b/assets/model_monitoring/components/generation_safety_quality/input_schema_adaptor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: gsq_input_schema_adaptor display_name: Input Schema Adaptor description: Adapt data to fit into GSQ component. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: production_dataset: @@ -35,8 +35,8 @@ conf: - azure-storage-file-datalake~=12.8.0 - azure-ai-ml~=1.26.3 - azureml-mlflow~=1.49.0 - - mltable~=1.3.0 - - azureml-fsspec~=1.0.0 + - mltable~=1.6.1 + - azureml-fsspec~=1.3.1 - fsspec~=2023.4.0 - numpy<2.0.0 name: momo-base-spark diff --git a/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml index 07a80d385a..c1f8dfa494 100644 --- a/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/genai_mdc_preprocessor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: genai_mdc_preprocessor display_name: GenAI MDC - Preprocessor description: Filters the raw span log based on the window provided, and aggregates it to trace level. -version: 0.0.28 +version: 0.0.29 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml index 95c86b4da3..5074803ac2 100644 --- a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_data_collector_preprocessor display_name: Model Data Collector - Preprocessor description: Filters the data based on the window provided. -version: 0.4.29 +version: 0.4.30 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml index 96358f208a..aa7006fa0e 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_azmon_metric_publisher/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_azmon_metric_publisher display_name: Model Monitor - Azure Monitor Metric Publisher description: Azure Monitor Publisher for the computed model monitor metrics. -version: 0.3.31 +version: 0.3.32 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml index bea674b797..40946e57a9 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_compute_histogram display_name: Model Monitor - Compute Histogram description: Compute a histogram given an input data and associated histogram buckets. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml index f1d18eb705..926d631c58 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_compute_histogram_buckets display_name: Model Monitor - Compute Histogram Buckets description: Compute histogram buckets given up to two datasets. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml index ba25c5f5cb..e5a41373c7 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_create_manifest display_name: Model Monitor - Create Manifest description: Creates the model monitor metric manifest. -version: 0.3.24 +version: 0.3.25 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml index 0c07c9735b..9b322230e2 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_data_joiner display_name: Model Monitor - Data Joiner description: Joins two data assets on the given columns for model monitor. -version: 0.3.27 +version: 0.3.28 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml index 713342d63e..8d124db6f2 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_evaluate_metrics_threshold display_name: Model Monitor - Evaluate Metrics Threshold description: Evaluate signal metrics against the threshold provided in the monitoring signal. -version: 0.3.32 +version: 0.3.33 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml index e24e594285..30c18ac735 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_feature_selector display_name: Model Monitor - Feature Selector description: Selects features to compute signal metrics on. -version: 0.3.28 +version: 0.3.29 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml index 722dcfe219..83f172e9cc 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_metric_outputter display_name: Model Monitor - Metric Outputter description: Output the computed model monitor metrics. -version: 0.3.35 +version: 0.3.36 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml index c1e828fbbe..1c1507eb1f 100644 --- a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml +++ b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_output_metrics display_name: Model Monitor - Output Metrics description: Output the computed model monitor metrics to the default datastore. -version: 0.3.32 +version: 0.3.33 is_deterministic: true code: ../../src/ diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml index 8c8b8f84de..0c9dcae963 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_correlation_test/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_correlation_test display_name: Action Analyzer - Correlation Test description: Perform correlation test on different groups to generate actions. -version: 0.0.21 +version: 0.0.22 is_deterministic: True inputs: data_with_action_metric_score: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml index 0b9b97cd39..67b5bdfada 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_identify_problem_traffic/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_identify_problem_traffic display_name: Action Analyzer - Identify Problem Traffic description: Separate bad queries into different groups. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: signal_output: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml index 658714a7fc..f054b6b3ae 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_metrics_calculation/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_metrics_calculation display_name: Action Analyzer - Metrics Calculation description: Calculate futher metrics for generating actions. -version: 0.0.21 +version: 0.0.22 is_deterministic: True inputs: data_with_groups: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml index 7bd80a94ab..6c15cddc1e 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/action_analyzer_output_actions/spec.yaml @@ -4,7 +4,7 @@ type: spark name: action_analyzer_output_actions display_name: Action Analyzer - Output Actions description: Merge and output actions. -version: 0.0.24 +version: 0.0.25 is_deterministic: True inputs: action_data: diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml index 42d2d8bb59..42e8f567eb 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_analyzer/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: model_monitor_action_analyzer display_name: Model Monitor - Action Analyzer description: Generate and output actions to the default datastore. -version: 0.0.25 +version: 0.0.26 is_deterministic: true inputs: signal_output: @@ -52,7 +52,7 @@ outputs: jobs: identify_problem_traffic: type: spark - component: azureml://registries/azureml/components/action_analyzer_identify_problem_traffic/versions/0.0.24 + component: azureml://registries/azureml/components/action_analyzer_identify_problem_traffic/versions/0.0.25 inputs: signal_output: type: uri_folder @@ -74,7 +74,7 @@ jobs: type: aml_token metrics_calculation: type: spark - component: azureml://registries/azureml/components/action_analyzer_metrics_calculation/versions/0.0.21 + component: azureml://registries/azureml/components/action_analyzer_metrics_calculation/versions/0.0.22 inputs: data_with_groups: type: mltable @@ -91,7 +91,7 @@ jobs: type: aml_token correlation_test: type: spark - component: azureml://registries/azureml/components/action_analyzer_correlation_test/versions/0.0.21 + component: azureml://registries/azureml/components/action_analyzer_correlation_test/versions/0.0.22 inputs: data_with_action_metric_score: type: mltable @@ -106,7 +106,7 @@ jobs: type: aml_token output_actions: type: spark - component: azureml://registries/azureml/components/action_analyzer_output_actions/versions/0.0.24 + component: azureml://registries/azureml/components/action_analyzer_output_actions/versions/0.0.25 inputs: action_data: type: mltable diff --git a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml index 467321060c..ec21779dde 100644 --- a/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml +++ b/assets/model_monitoring/components/model_monitor_action_analyzer/model_monitor_action_detector/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_monitor_action_detector display_name: Model Monitor - Action Detector description: Generate and output actions -version: 0.0.17 +version: 0.0.18 is_deterministic: true inputs: signal_output: diff --git a/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml b/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml index 7650b59b1c..8bfe429cff 100644 --- a/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: model_performance_compute_metrics display_name: Model Performance - Compute Metrics description: Compute model performance metrics leveraged by the model performance monitor. -version: 0.0.26 +version: 0.0.27 is_deterministic: true code: ../../src diff --git a/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml b/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml index 766d3608f8..da8e745b64 100644 --- a/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/model_performance/model_performance_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: model_performance_signal_monitor display_name: Model Performance - Signal Monitor description: Computes the model performance -version: 0.0.28 +version: 0.0.29 is_deterministic: true inputs: task: @@ -57,7 +57,7 @@ outputs: jobs: compute_metrics: type: spark - component: azureml://registries/azureml/components/model_performance_compute_metrics/versions/0.0.26 + component: azureml://registries/azureml/components/model_performance_compute_metrics/versions/0.0.27 inputs: task: ${{parent.inputs.task}} baseline_data_target_column: ${{parent.inputs.baseline_data_target_column}} @@ -78,7 +78,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.35 + component: azureml://registries/azureml/components/model_monitor_metric_outputter/versions/0.3.36 inputs: signal_metrics: type: mltable @@ -99,7 +99,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml b/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml index b2cdafdf3e..094db6d9f4 100644 --- a/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml +++ b/assets/model_monitoring/components/prediction_drift/prediction_drift_signal_monitor/spec.yaml @@ -4,7 +4,7 @@ type: pipeline name: prediction_drift_signal_monitor display_name: Prediction Drift - Signal Monitor description: Computes the prediction drift between a baseline and a target data assets. -version: 0.4.30 +version: 0.4.31 is_deterministic: true inputs: @@ -57,7 +57,7 @@ outputs: jobs: feature_selection: type: spark - component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.28 + component: azureml://registries/azureml/components/model_monitor_feature_selector/versions/0.3.29 inputs: input_data_1: type: mltable @@ -77,7 +77,7 @@ jobs: type: aml_token compute_drift_metrics: type: spark - component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.33 + component: azureml://registries/azureml/components/data_drift_compute_metrics/versions/0.3.34 inputs: production_dataset: type: mltable @@ -104,7 +104,7 @@ jobs: type: aml_token compute_histogram_buckets: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram_buckets/versions/0.3.28 inputs: input_data_1: type: mltable @@ -124,7 +124,7 @@ jobs: type: aml_token compute_baseline_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -144,7 +144,7 @@ jobs: type: aml_token compute_target_histogram: type: spark - component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.27 + component: azureml://registries/azureml/components/model_monitor_compute_histogram/versions/0.3.28 inputs: input_data: type: mltable @@ -164,7 +164,7 @@ jobs: type: aml_token output_signal_metrics: type: spark - component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_output_metrics/versions/0.3.33 inputs: signal_metrics: type: mltable @@ -191,7 +191,7 @@ jobs: type: aml_token evaluate_metric_thresholds: type: spark - component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.32 + component: azureml://registries/azureml/components/model_monitor_evaluate_metrics_threshold/versions/0.3.33 inputs: signal_metrics: type: mltable diff --git a/assets/model_monitoring/components/tests/e2e/conftest.py b/assets/model_monitoring/components/tests/e2e/conftest.py index d47ac2ddcf..3dba18efa5 100644 --- a/assets/model_monitoring/components/tests/e2e/conftest.py +++ b/assets/model_monitoring/components/tests/e2e/conftest.py @@ -13,9 +13,11 @@ from azure.ai.ml.constants import AssetTypes from azure.ai.ml.operations._run_history_constants import RunHistoryConstants, JobStatus from azure.ai.ml.entities import Job, Data +from azure.core.exceptions import ResourceNotFoundError from azure.identity import AzureCliCredential import pytest + from tests.e2e.utils.io_utils import ( write_to_yaml, load_from_yaml, @@ -56,14 +58,21 @@ def _watch_file(file: str, timeout_in_seconds): def _create_data_asset(ml_client: MLClient, name: str, path: str, type: AssetTypes): - my_data = Data( - path=path, - type=type, - name=name, - version='1' - ) - print(f"Creating a {type} data asset with name '{name}'") - ml_client.data.create_or_update(my_data) + try: + ml_client.data.get(name=name, version='1') + print(f"Data asset '{name}' version '1' already exists.") + except ResourceNotFoundError: + my_data = Data( + path=path, + type=type, + name=name, + version='1' + ) + print(f"Creating a {type} data asset with name '{name}', version '1'") + ml_client.data.create_or_update(my_data) + except Exception as ex: + print(f"Unexpected error: {ex}") + raise @pytest.fixture(scope="session") diff --git a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py index afd190fecc..15f0d78fa1 100644 --- a/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py +++ b/assets/model_monitoring/components/tests/e2e/test_generation_safety_quality_e2e.py @@ -53,6 +53,7 @@ def _generation_safety_quality_signal_monitor_e2e(): completion_column_name=completion_column_name, ground_truth_column_name=ground_truth_column_name, context_column_name=context_column_name, + instance_type="standard_e8s_v3", ) return { "signal_output": generation_safety_quality_signal_monitor_output.outputs.signal_output @@ -60,7 +61,6 @@ def _generation_safety_quality_signal_monitor_e2e(): pipeline_job = _generation_safety_quality_signal_monitor_e2e() pipeline_job.outputs.signal_output = Output(type="uri_folder", mode="direct") - pipeline_job = submit_pipeline_job( pipeline_job, experiment_name, expect_failure ) diff --git a/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml b/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml index 07c65e989c..338c5322cd 100644 --- a/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml +++ b/assets/model_monitoring/components/token_statistics/token_statistics_compute_metrics/spec.yaml @@ -4,7 +4,7 @@ type: spark name: token_statistics_compute_metrics display_name: Token Statistics - Compute Metrics description: Compute token statistics metrics. -version: 0.0.23 +version: 0.0.24 is_deterministic: true inputs: