From 93d0987068074fd2c82eb0f75096bc7d703b712f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 08:22:05 +0000 Subject: [PATCH 1/4] Initial plan From 58bfb64696d441583d4004cb7d51d4e14ea2252e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 08:25:58 +0000 Subject: [PATCH 2/4] Update AutoML metric documentation with full list and documentation link Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com> --- flaml/automl/automl.py | 4 +++ .../docs/Use-Cases/Task-Oriented-AutoML.md | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py index 5cf2d71727..fb6731c182 100644 --- a/flaml/automl/automl.py +++ b/flaml/automl/automl.py @@ -118,6 +118,8 @@ def __init__(self, **settings): e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_weighted', 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted', 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2', 'mape'. Default is 'auto'. + For a full list of supported built-in metrics, please refer to + https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#optimization-metric If passing a customized metric function, the function needs to have the following input arguments: @@ -1287,6 +1289,8 @@ def fit( e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_weighted', 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted', 'f1', 'micro_f1', 'macro_f1', 'log_loss', 'mae', 'mse', 'r2', 'mape'. Default is 'auto'. + For a full list of supported built-in metrics, please refer to + https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#optimization-metric If passing a customized metric function, the function needs to have the following input arguments: diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.md index c51c6a3c11..5043b56245 100644 --- a/website/docs/Use-Cases/Task-Oriented-AutoML.md +++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md @@ -69,6 +69,39 @@ The optimization metric is specified via the `metric` argument. It can be either - 'ap': minimize 1 - average_precision_score. - 'ndcg': minimize 1 - ndcg_score. - 'ndcg@k': minimize 1 - ndcg_score@k. k is an integer. + - 'pr_auc': minimize 1 - precision-recall AUC score. (Spark-specific) + - 'var': minimize variance. (Spark-specific) + +- Built-in HuggingFace metrics (for NLP tasks). + - 'accuracy': minimize 1 - accuracy. + - 'bertscore': minimize 1 - BERTScore. + - 'bleu': minimize 1 - BLEU score. + - 'bleurt': minimize 1 - BLEURT score. + - 'cer': minimize character error rate. + - 'chrf': minimize ChrF score. + - 'code_eval': minimize 1 - code evaluation score. + - 'comet': minimize 1 - COMET score. + - 'competition_math': minimize 1 - competition math score. + - 'coval': minimize 1 - CoVal score. + - 'cuad': minimize 1 - CUAD score. + - 'f1': minimize 1 - F1 score. + - 'gleu': minimize 1 - GLEU score. + - 'google_bleu': minimize 1 - Google BLEU score. + - 'matthews_correlation': minimize 1 - Matthews correlation coefficient. + - 'meteor': minimize 1 - METEOR score. + - 'pearsonr': minimize 1 - Pearson correlation coefficient. + - 'precision': minimize 1 - precision. + - 'recall': minimize 1 - recall. + - 'rouge': minimize 1 - ROUGE score. + - 'rouge1': minimize 1 - ROUGE-1 score. + - 'rouge2': minimize 1 - ROUGE-2 score. + - 'sacrebleu': minimize 1 - SacreBLEU score. + - 'sari': minimize 1 - SARI score. + - 'seqeval': minimize 1 - SeqEval score. + - 'spearmanr': minimize 1 - Spearman correlation coefficient. + - 'ter': minimize translation error rate. + - 'wer': minimize word error rate. + - User-defined function. A customized metric function that requires the following (input) signature, and returns the input config’s value in terms of the metric you want to minimize, and a dictionary of auxiliary information at your choice: From 87b5faa54d643c580868cf7d091d9d6e589c4ba1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 08:28:35 +0000 Subject: [PATCH 3/4] Apply black and mdformat formatting to code and documentation Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com> --- flaml/automl/automl.py | 10 ++++------ website/docs/Use-Cases/Task-Oriented-AutoML.md | 8 +++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py index fb6731c182..abb3e99757 100644 --- a/flaml/automl/automl.py +++ b/flaml/automl/automl.py @@ -1737,9 +1737,7 @@ def cv_score_agg_func(val_loss_folds, log_metrics_folds): self._mlflow_logging = ( False if mlflow is None - else self._settings.get("mlflow_logging") - if mlflow_logging is None - else mlflow_logging + else self._settings.get("mlflow_logging") if mlflow_logging is None else mlflow_logging ) fit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator") self._state.fit_kwargs_by_estimator = fit_kwargs_by_estimator.copy() # shallow copy of fit_kwargs_by_estimator @@ -1981,9 +1979,9 @@ def is_to_reverse_metric(metric, task): this_estimator_kwargs.update( self._state.fit_kwargs ) # update the shallow copy of fit_kwargs to fit_kwargs_by_estimator - self._state.fit_kwargs_by_estimator[ - estimator_name - ] = this_estimator_kwargs # set self._state.fit_kwargs_by_estimator[estimator_name] to the update, so only self._state.fit_kwargs_by_estimator will be updated + self._state.fit_kwargs_by_estimator[estimator_name] = ( + this_estimator_kwargs # set self._state.fit_kwargs_by_estimator[estimator_name] to the update, so only self._state.fit_kwargs_by_estimator will be updated + ) else: self._state.fit_kwargs_by_estimator[estimator_name] = self._state.fit_kwargs diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.md index 5043b56245..27cc153fab 100644 --- a/website/docs/Use-Cases/Task-Oriented-AutoML.md +++ b/website/docs/Use-Cases/Task-Oriented-AutoML.md @@ -50,6 +50,7 @@ If users provide the minimal inputs only, `AutoML` uses the default settings for The optimization metric is specified via the `metric` argument. It can be either a string which refers to a built-in metric, or a user-defined function. - Built-in metric. + - 'accuracy': 1 - accuracy as the corresponding metric to minimize. - 'log_loss': default metric for multiclass classification. - 'r2': 1 - r2_score as the corresponding metric to minimize. Default metric for regression. @@ -73,6 +74,7 @@ The optimization metric is specified via the `metric` argument. It can be either - 'var': minimize variance. (Spark-specific) - Built-in HuggingFace metrics (for NLP tasks). + - 'accuracy': minimize 1 - accuracy. - 'bertscore': minimize 1 - BERTScore. - 'bleu': minimize 1 - BLEU score. @@ -164,7 +166,7 @@ The estimator list can contain one or more estimator names, each corresponding t - Built-in estimator. - 'lgbm': LGBMEstimator for task "classification", "regression", "rank", "ts_forecast" and "ts_forecast_classification". Hyperparameters: n_estimators, num_leaves, min_child_samples, learning_rate, log_max_bin (logarithm of (max_bin + 1) with base 2), colsample_bytree, reg_alpha, reg_lambda. - 'xgboost': XGBoostSkLearnEstimator for task "classification", "regression", "rank", "ts_forecast" and "ts_forecast_classification". Hyperparameters: n_estimators, max_leaves, min_child_weight, learning_rate, subsample, colsample_bylevel, colsample_bytree, reg_alpha, reg_lambda. - - 'xgb_limitdepth': XGBoostLimitDepthEstimator for task "classification", "regression", "rank", "ts_forecast" and "ts_forecast_classification". Hyperparameters: n_estimators, max_depth, min_child_weight, learning_rate, subsample, colsample_bylevel, colsample_bytree, reg_alpha, reg_lambda. + - 'xgb_limitdepth': XGBoostLimitDepthEstimator for task "classification", "regression", "rank", "ts_forecast" and "ts_forecast_classification". Hyperparameters: n_estimators, max_depth, min_child_weight, learning_rate, subsample, colsample_bylevel, colsample_bytree, reg_alpha, reg_lambda. - 'rf': RandomForestEstimator for task "classification", "regression", "ts_forecast" and "ts_forecast_classification". Hyperparameters: n_estimators, max_features, max_leaves, criterion (for classification only). Starting from v1.1.0, it uses a fixed random_state by default. - 'extra_tree': ExtraTreesEstimator for task "classification", "regression", "ts_forecast" and "ts_forecast_classification". Hyperparameters: n_estimators, max_features, max_leaves, criterion (for classification only). Starting from v1.1.0, @@ -448,7 +450,7 @@ For both classification and regression tasks more advanced split configurations More in general, `split_type` can also be set as a custom splitter object, when `eval_method="cv"`. It needs to be an instance of a derived class of scikit-learn [KFold](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.KFold.html#sklearn.model_selection.KFold) -and have `split` and `get_n_splits` methods with the same signatures. To disable shuffling, the splitter instance must contain the attribute `shuffle=False`. +and have `split` and `get_n_splits` methods with the same signatures. To disable shuffling, the splitter instance must contain the attribute `shuffle=False`. ### Parallel tuning @@ -726,7 +728,7 @@ If you want to get a sense of how much time is needed to find the best model, yo > INFO - Estimated sufficient time budget=145194s. Estimated necessary time budget=2118s. -> INFO - at 2.6s, estimator lgbm's best error=0.4459, best estimator lgbm's best error=0.4459 +> INFO - at 2.6s, estimator lgbm's best error=0.4459, best estimator lgbm's best error=0.4459 You will see that the time to finish the first and cheapest trial is 2.6 seconds. The estimated necessary time budget is 2118 seconds, and the estimated sufficient time budget is 145194 seconds. Note that this is only an estimated range to help you decide your budget. From bd90f6d6400f36bf2009dcca61dc35ccc650d03a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Jan 2026 00:01:54 +0000 Subject: [PATCH 4/4] Apply pre-commit formatting fixes Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com> --- flaml/automl/automl.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py index ad1340982b..77178202e1 100644 --- a/flaml/automl/automl.py +++ b/flaml/automl/automl.py @@ -2098,7 +2098,9 @@ def cv_score_agg_func(val_loss_folds, log_metrics_folds): self._mlflow_logging = ( False if mlflow is None - else self._settings.get("mlflow_logging") if mlflow_logging is None else mlflow_logging + else self._settings.get("mlflow_logging") + if mlflow_logging is None + else mlflow_logging ) fit_kwargs_by_estimator = fit_kwargs_by_estimator or self._settings.get("fit_kwargs_by_estimator") self._state.fit_kwargs_by_estimator = fit_kwargs_by_estimator.copy() # shallow copy of fit_kwargs_by_estimator @@ -2340,9 +2342,9 @@ def is_to_reverse_metric(metric, task): this_estimator_kwargs.update( self._state.fit_kwargs ) # update the shallow copy of fit_kwargs to fit_kwargs_by_estimator - self._state.fit_kwargs_by_estimator[estimator_name] = ( - this_estimator_kwargs # set self._state.fit_kwargs_by_estimator[estimator_name] to the update, so only self._state.fit_kwargs_by_estimator will be updated - ) + self._state.fit_kwargs_by_estimator[ + estimator_name + ] = this_estimator_kwargs # set self._state.fit_kwargs_by_estimator[estimator_name] to the update, so only self._state.fit_kwargs_by_estimator will be updated else: self._state.fit_kwargs_by_estimator[estimator_name] = self._state.fit_kwargs